segmenting image - the pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Searching for objects in mock images from the ArcsFinding Challenge\n",
    "\n",
    "What I we are going to do now is to build the pipeline from some methods explored in the previous notebook ``segmenting image``.\n",
    "\n",
    "The pipeline should get one image --monochromatic image-- and output (*i*) a table (~pandas.DataFrame) with morphological quantidies along the lines and the statistical measurements as columns, (*ii*) a panel with figures of each step of the processing (including the input image). The features table should also be written as a CSV file.\n",
    "\n",
    "The input files are named as \"``imageEUC_VIS-``*id*``.fits``\".\n",
    "\"*id*\" is a number, unique to each image.\n",
    "\n",
    "The name of the output files should be \"*id*``-featureMatrix.csv``\" and \"*id*``-partialOuts.png``\".\n",
    "\n",
    "The pipeline goes like:\n",
    "* image normalization\n",
    "* pre-processing\n",
    " * smoothing\n",
    "* segmentation\n",
    " * background threshold\n",
    " * contours definition\n",
    " * local maxima\n",
    " * regions mask\n",
    " * objects identification\n",
    "* post-processing\n",
    " * features extraction\n",
    "* output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class PipelineBase:\n",
    "\n",
    "    _inputFileFormat = 'imageEUC_VIS-{0}.fits'\n",
    "    _outputTableFileFormat = 'featuresMatrix-{0}.csv'\n",
    "    _outputPlotFileFormat = 'partialOuts-{0}.png'\n",
    "    \n",
    "    def __init__(self,id):\n",
    "        self.id = id\n",
    "\n",
    "    def read_image(self):\n",
    "        assert False, \"Base class, method not implemented\"\n",
    "        \n",
    "    def normalize(self):\n",
    "        assert False, \"Base class, method not implemented\"\n",
    "        \n",
    "    def smooth(self):\n",
    "        assert False, \"Base class, method not implemented\"\n",
    "\n",
    "    def estimate_threshold(self):\n",
    "        assert False, \"Base class, method not implemented\"\n",
    "    \n",
    "    def find_maxima(self):\n",
    "        assert False, \"Base class, method not implemented\"\n",
    "    \n",
    "    def find_countours(self):\n",
    "        assert False, \"Base class, method not implemented\"\n",
    "    \n",
    "    def define_objects(self):\n",
    "        assert False, \"Base class, method not implemented\"\n",
    "    \n",
    "    def compute_features(self):\n",
    "        assert False, \"Base class, method not implemented\"\n",
    "    \n",
    "    def write_table(self):\n",
    "        assert False, \"Base class, method not implemented\"\n",
    "    \n",
    "    def write_plot(self):\n",
    "        assert False, \"Base class, method not implemented\"\n",
    "    \n",
    "\n",
    "    \n",
    "def normalize(img,unit=1):\n",
    "    \"\"\"\n",
    "    Normalize 'img' to 'unit'\n",
    "    \"\"\"\n",
    "    \n",
    "    return unit * (img - img.min())/(img.max() - img.min())\n",
    "\n",
    "def gaussian(img, sigma=[3,3]):\n",
    "    \"\"\"\n",
    "    Simple gaussian filter\n",
    "    \n",
    "    Input:\n",
    "     - img <ndarray>\n",
    "     - sigma <[int,int]> : sigma window\n",
    "    \n",
    "    Output:\n",
    "     - <ndarray>\n",
    "    \n",
    "    ---\n",
    "    \"\"\"\n",
    "    import scipy.ndimage as ndi\n",
    "    return ndi.gaussian_filter(img,sigma)\n",
    "\n",
    "def histmax(img):\n",
    "    \"\"\"\n",
    "    Maximum histogram value for threshold estimation\n",
    "    \"\"\"\n",
    "    import numpy as np\n",
    "    nbins=1000\n",
    "    imhist,bins = np.histogram(img.flatten(),nbins,normed=True);\n",
    "\n",
    "    return bins[np.argmax(imhist)] + np.std(img)\n",
    "\n",
    "def mask_contours(contours, img_shape):\n",
    "    import numpy as np\n",
    "    \n",
    "    def define_regions(contours):\n",
    "        '''\n",
    "        Return a list of regions (~matplotlib.path.Path), for each entry in 'country_map'\n",
    "\n",
    "        'country_map' is expected to provide columns 'lons' and 'lats',\n",
    "        providing lists of coordinates defining the respective country\n",
    "        as polygon(s). If 'country_map' has multiple lines, multiple\n",
    "        \"Paths\" will be created (for each line/polygon).\n",
    "        '''\n",
    "        from matplotlib.path import Path\n",
    "        import numpy as np\n",
    "        regions_path = []\n",
    "        for n, contour in enumerate(contours):\n",
    "            regions_path.append( Path( contour ) )\n",
    "        return regions_path\n",
    "\n",
    "    # Create vertex coordinates for each grid cell...\n",
    "    # (<0,0> is at the top left of the grid in this system)\n",
    "    ny,nx = img_shape\n",
    "    x, y = np.meshgrid(np.arange(nx), np.arange(ny))\n",
    "    x, y = x.flatten(), y.flatten()\n",
    "\n",
    "    grid = np.vstack((y,x)).T\n",
    "\n",
    "    mask = np.zeros(img_shape).astype(bool)\n",
    "    paths = define_regions(contours)\n",
    "    for path in paths:\n",
    "        _mask = path.contains_points(grid)\n",
    "        mask = mask + _mask.reshape((ny,nx))\n",
    "    return mask\n",
    "\n",
    "def region_growing(img,x,y,thresh):\n",
    "    \"\"\"\n",
    "    Segment using a Region Growing algorithm\n",
    "    \n",
    "    Input:\n",
    "     - img  ndarray : Image array (ndim=2)\n",
    "     - x        int : Seed x position\n",
    "     - y        int : Seed y position\n",
    "     - thresh float : Threshold value for limiting the grow\n",
    "    \n",
    "    Output:\n",
    "     - region  : Region grown around given 'x,y'\n",
    "    \n",
    "    ---\n",
    "    \"\"\"\n",
    "    from scipy import ndimage as ndi\n",
    "    import numpy as np\n",
    "    \n",
    "    x_o = x\n",
    "    y_o = y\n",
    "    flag = True\n",
    "\n",
    "    # Initialize region with the seed point\n",
    "    region = np.zeros(img.shape,dtype=np.bool)\n",
    "    reg_old = (region==flag)\n",
    "\n",
    "    if (img[y_o,x_o] < thresh):\n",
    "        return region\n",
    "    \n",
    "    region[y_o,x_o] = flag\n",
    "    reg_cur = (region==flag)\n",
    "\n",
    "    # For future (loop) morphological operations...\n",
    "    strct_elem = ndi.generate_binary_structure(2,2)\n",
    "\n",
    "    # While region stills changes (grow), do...\n",
    "    while not np.all(region == reg_old):\n",
    "        \n",
    "        reg_old = region.copy()\n",
    "\n",
    "        # Define pixel neighbours\n",
    "        tmp = ndi.binary_dilation(region,strct_elem, 1)\n",
    "        neigbors = tmp - region\n",
    "        inds = np.where(neigbors)\n",
    "\n",
    "        # Check for the new neighbors; do they fullfil requirements?\n",
    "        #region[np.where(region[inds]>=thresh)] = flag\n",
    "        for y_i,x_i in zip(*inds):\n",
    "            if (img[y_i,x_i] >= thresh):\n",
    "                region[y_i,x_i] = flag\n",
    "\n",
    "    return region\n",
    "\n",
    "def objects_properties(img_segm, img_smooth):\n",
    "    import numpy as np\n",
    "    from pandas import DataFrame\n",
    "    from skimage.measure import regionprops\n",
    "    reg_props = regionprops(img_segm,img_smooth)\n",
    "\n",
    "    d_props = {}\n",
    "    l_props = []\n",
    "    d2_props = {}\n",
    "    l2_props = []\n",
    "    for props in reg_props:\n",
    "        label = props.label\n",
    "        vals = []\n",
    "        vals2 = []\n",
    "        for prop in props:\n",
    "            val = props[prop]\n",
    "            if np.isscalar(val):\n",
    "                vals.append(val)\n",
    "                l_props.append(prop)\n",
    "            else:\n",
    "                vals2.append(val)\n",
    "                l2_props.append(prop)\n",
    "        d_props[label] = vals\n",
    "        d2_props[label] = vals2\n",
    "    # 'df' contain morphological parameters from each identified object\n",
    "    index = l_props[:len(vals)]\n",
    "    df = DataFrame(d_props,index=index).transpose()\n",
    "    return df\n",
    "\n",
    "def plot_panelImages(img,img_smooth,img_maxima,contours,img_segm):\n",
    "    from matplotlib import pyplot as plt\n",
    "\n",
    "    num_panels = 4\n",
    "    fig, axes = plt.subplots(ncols=num_panels, figsize=(12, 3), sharex=True, sharey=True,\n",
    "                             subplot_kw={'adjustable': 'box-forced'})\n",
    "    ax = axes.ravel()\n",
    "\n",
    "    _ = ax[0].imshow(img, cmap=plt.cm.gray, interpolation='nearest')\n",
    "    ax[0].set_title('Image (original)')\n",
    "\n",
    "    _ = ax[1].imshow(-img_smooth, cmap=plt.cm.gray, interpolation='nearest')\n",
    "    ax[1].set_title('Image smooth')\n",
    "\n",
    "    ax[2].imshow(img_maxima, cmap=plt.cm.gray)\n",
    "    for n, contour in enumerate(contours):\n",
    "        ax[2].plot(contour[:, 1], contour[:, 0], linewidth=2, color='red')\n",
    "    ax[2].set_title('Contours and maxima')\n",
    "    ax[2].set_xticks([])\n",
    "    ax[2].set_yticks([])\n",
    "\n",
    "    im = ax[3].imshow(img_segm, cmap=plt.cm.spectral, interpolation='nearest')\n",
    "    ax[3].set_title('Segmented objects')\n",
    "\n",
    "    cax = fig.add_axes([0.91, 0.2, 0.01, 0.6])\n",
    "    import numpy as np\n",
    "    vals = np.unique(img_segm)\n",
    "    fig.colorbar(im, cax=cax, ticks=vals)\n",
    "\n",
    "    for a in ax:\n",
    "        a.set_axis_off()\n",
    "    \n",
    "    return plt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Pipeline(PipelineBase):\n",
    "    \n",
    "    def read_image(self,dir=None):\n",
    "        from astropy.io import fits\n",
    "        filename = self._inputFileFormat.format(self.id)\n",
    "        if dir != None:\n",
    "            from os import path\n",
    "            filename = path.join(dir,filename)\n",
    "        \n",
    "        img = fits.open(filename)[0].data\n",
    "        return img\n",
    "    \n",
    "    def normalize(self,img):\n",
    "        return normalize(img)\n",
    "    \n",
    "    def smooth(self,img):\n",
    "        return gaussian(img,[2,2])\n",
    "    \n",
    "    def estimate_threshold(self,img):\n",
    "        return histmax(img)\n",
    "    \n",
    "    def find_contours(self,img,thresh):\n",
    "        from skimage import measure\n",
    "        contours = measure.find_contours(img,thresh)\n",
    "        return contours\n",
    "    \n",
    "    def find_maxima(self,img):\n",
    "        from skimage.feature import peak_local_max\n",
    "        img_local_maxi = peak_local_max(img, min_distance=2, exclude_border=False, indices=False)\n",
    "        return img_local_maxi\n",
    "    \n",
    "    def define_objects(self,img,thresh,maxima):\n",
    "        import numpy as np\n",
    "        if maxima.shape == img.shape:\n",
    "            _lmi = np.where(maxima)\n",
    "            local_maxi_yx = zip(_lmi[0],_lmi[1])\n",
    "        else:\n",
    "            local_maxi_yx = maxima\n",
    "        img_reg = np.zeros(img.shape,dtype=int)\n",
    "        for n,yx in enumerate(local_maxi_yx):\n",
    "            y,x = yx\n",
    "            reg = region_growing(img,x,y,thresh)#+0.1)\n",
    "            _img = reg.astype(int) * n\n",
    "            _idx = np.where(reg)\n",
    "            img_reg[_idx] = _img[_idx]\n",
    "        return img_reg\n",
    "\n",
    "    def compute_features(self, img_segm, img_smooth):\n",
    "        import pandas\n",
    "        try:\n",
    "            tab_objects = objects_properties(img_segm, img_smooth)\n",
    "            _features_matrix_T = tab_objects.describe()\n",
    "            _skew_cols = tab_objects.skew()\n",
    "            _kurt_cols = tab_objects.kurtosis()\n",
    "            _df = pandas.DataFrame([_skew_cols,_kurt_cols],index=['skew','kurtosis'],columns=tab_objects.columns)\n",
    "            features_matrix = pandas.concat([_features_matrix_T,_df]).transpose()\n",
    "        except:\n",
    "            features_matrix = None\n",
    "\n",
    "        return features_matrix\n",
    "    \n",
    "    def write_table(self,df,dir='output'):\n",
    "        import os\n",
    "#         _dir = os.path.join(dir,str(self.id))\n",
    "        _dir = dir\n",
    "        mkdir(_dir)\n",
    "        filename = self._outputTableFileFormat.format(self.id)\n",
    "        filename = os.path.join(_dir,filename)\n",
    "        df.to_csv(filename)\n",
    "        \n",
    "    def write_plot(self,plot,dir='output'):\n",
    "        import os\n",
    "#         _dir = os.path.join(dir,str(self.id))\n",
    "        _dir = dir\n",
    "        mkdir(_dir)\n",
    "        filename = self._outputPlotFileFormat.format(self.id)\n",
    "        filename = os.path.join(_dir,filename)\n",
    "        plot.savefig(filename)\n",
    "        \n",
    "def mkdir(path):\n",
    "    import os\n",
    "    _pathblocks = path.split(os.path.sep)\n",
    "    _path = ''\n",
    "    for _dir_ in _pathblocks:\n",
    "        _path = os.path.join(_path,_dir_)\n",
    "        if not os.path.exists(_path):\n",
    "            os.mkdir(_path)\n",
    "    return os.path.join(_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chbrandt/.conda/envs/booq-dev/lib/python2.7/site-packages/ipykernel/__main__.py:151: DeprecationWarning: numpy boolean subtract, the `-` operator, is deprecated, use the bitwise_xor, the `^` operator, or the logical_xor function instead.\n"
     ]
    }
   ],
   "source": [
    "def run_pipeline(id,input_dir='data/'):\n",
    "    ppline = Pipeline(id)\n",
    "\n",
    "    # read data\n",
    "    img = ppline.read_image(dir=input_dir)\n",
    "\n",
    "    # normalize image\n",
    "    img = ppline.normalize(img)\n",
    "\n",
    "    # smooth the image\n",
    "    img_smooth = ppline.smooth(img)\n",
    "\n",
    "    # get the background threshold\n",
    "    back_thresh = ppline.estimate_threshold(img)\n",
    "\n",
    "    # define the countours\n",
    "    contours = ppline.find_contours(img_smooth, back_thresh)\n",
    "\n",
    "    # use contours as the valid/candidate regions\n",
    "    _mask = mask_contours(contours, img.shape)\n",
    "\n",
    "    # find the local maxima\n",
    "    img_maxima = ppline.find_maxima(img_smooth * _mask)\n",
    "\n",
    "    # better define the objects\n",
    "    img_segm = ppline.define_objects(img, back_thresh, img_maxima)\n",
    "\n",
    "    # generate the image processing plots\n",
    "    plot = plot_panelImages(img,img_smooth,img_maxima,contours,img_segm)\n",
    "    ppline.write_plot(plot)\n",
    "    \n",
    "    # compute the features\n",
    "    features_matrix = ppline.compute_features(img_segm, img_smooth)\n",
    "    if not features_matrix is None:\n",
    "        ppline.write_table(features_matrix)\n",
    "\n",
    "    return features_matrix\n",
    "\n",
    "\n",
    "# read/define id\n",
    "id = 100003\n",
    "\n",
    "features_matrix = run_pipeline(id,'data/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "      <th>skew</th>\n",
       "      <th>kurtosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>area</th>\n",
       "      <td>6.0</td>\n",
       "      <td>172.166667</td>\n",
       "      <td>169.641288</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>53.250000</td>\n",
       "      <td>103.500000</td>\n",
       "      <td>291.750000</td>\n",
       "      <td>422.000000</td>\n",
       "      <td>0.812830</td>\n",
       "      <td>-1.419060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convex_area</th>\n",
       "      <td>6.0</td>\n",
       "      <td>711.500000</td>\n",
       "      <td>1086.728071</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>61.000000</td>\n",
       "      <td>163.000000</td>\n",
       "      <td>895.750000</td>\n",
       "      <td>2763.000000</td>\n",
       "      <td>1.805071</td>\n",
       "      <td>2.969080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eccentricity</th>\n",
       "      <td>6.0</td>\n",
       "      <td>0.656375</td>\n",
       "      <td>0.247219</td>\n",
       "      <td>0.329209</td>\n",
       "      <td>0.540171</td>\n",
       "      <td>0.593026</td>\n",
       "      <td>0.846596</td>\n",
       "      <td>0.969315</td>\n",
       "      <td>0.263787</td>\n",
       "      <td>-1.189019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>equivalent_diameter</th>\n",
       "      <td>6.0</td>\n",
       "      <td>13.093467</td>\n",
       "      <td>7.571304</td>\n",
       "      <td>4.068429</td>\n",
       "      <td>8.223192</td>\n",
       "      <td>11.248391</td>\n",
       "      <td>19.012782</td>\n",
       "      <td>23.179885</td>\n",
       "      <td>0.380733</td>\n",
       "      <td>-1.657971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>euler_number</th>\n",
       "      <td>6.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>extent</th>\n",
       "      <td>6.0</td>\n",
       "      <td>0.447165</td>\n",
       "      <td>0.301338</td>\n",
       "      <td>0.082005</td>\n",
       "      <td>0.189305</td>\n",
       "      <td>0.488044</td>\n",
       "      <td>0.664015</td>\n",
       "      <td>0.812500</td>\n",
       "      <td>-0.203864</td>\n",
       "      <td>-1.892205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>filled_area</th>\n",
       "      <td>6.0</td>\n",
       "      <td>172.166667</td>\n",
       "      <td>169.641288</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>53.250000</td>\n",
       "      <td>103.500000</td>\n",
       "      <td>291.750000</td>\n",
       "      <td>422.000000</td>\n",
       "      <td>0.812830</td>\n",
       "      <td>-1.419060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <td>6.0</td>\n",
       "      <td>3.666667</td>\n",
       "      <td>2.160247</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.250000</td>\n",
       "      <td>3.500000</td>\n",
       "      <td>4.750000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>0.462910</td>\n",
       "      <td>-0.300000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>major_axis_length</th>\n",
       "      <td>6.0</td>\n",
       "      <td>38.370325</td>\n",
       "      <td>44.316619</td>\n",
       "      <td>4.318667</td>\n",
       "      <td>8.900622</td>\n",
       "      <td>14.965102</td>\n",
       "      <td>63.585279</td>\n",
       "      <td>109.301946</td>\n",
       "      <td>1.122066</td>\n",
       "      <td>-0.671105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max_intensity</th>\n",
       "      <td>6.0</td>\n",
       "      <td>0.340745</td>\n",
       "      <td>0.124014</td>\n",
       "      <td>0.190913</td>\n",
       "      <td>0.284893</td>\n",
       "      <td>0.334849</td>\n",
       "      <td>0.348291</td>\n",
       "      <td>0.562848</td>\n",
       "      <td>1.142450</td>\n",
       "      <td>2.509521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_intensity</th>\n",
       "      <td>6.0</td>\n",
       "      <td>0.219547</td>\n",
       "      <td>0.030212</td>\n",
       "      <td>0.179395</td>\n",
       "      <td>0.204378</td>\n",
       "      <td>0.212181</td>\n",
       "      <td>0.243441</td>\n",
       "      <td>0.257887</td>\n",
       "      <td>0.219149</td>\n",
       "      <td>-1.226143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min_intensity</th>\n",
       "      <td>6.0</td>\n",
       "      <td>0.146734</td>\n",
       "      <td>0.013942</td>\n",
       "      <td>0.134497</td>\n",
       "      <td>0.135899</td>\n",
       "      <td>0.143656</td>\n",
       "      <td>0.152434</td>\n",
       "      <td>0.169924</td>\n",
       "      <td>0.941806</td>\n",
       "      <td>0.040328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>minor_axis_length</th>\n",
       "      <td>6.0</td>\n",
       "      <td>15.843706</td>\n",
       "      <td>13.173532</td>\n",
       "      <td>3.679465</td>\n",
       "      <td>8.046784</td>\n",
       "      <td>12.039514</td>\n",
       "      <td>18.343355</td>\n",
       "      <td>40.143232</td>\n",
       "      <td>1.533146</td>\n",
       "      <td>2.527297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orientation</th>\n",
       "      <td>6.0</td>\n",
       "      <td>0.481434</td>\n",
       "      <td>0.706937</td>\n",
       "      <td>-0.785398</td>\n",
       "      <td>0.359977</td>\n",
       "      <td>0.544868</td>\n",
       "      <td>0.961182</td>\n",
       "      <td>1.185966</td>\n",
       "      <td>-1.251893</td>\n",
       "      <td>2.013372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>perimeter</th>\n",
       "      <td>6.0</td>\n",
       "      <td>100.797306</td>\n",
       "      <td>104.599710</td>\n",
       "      <td>10.242641</td>\n",
       "      <td>26.503048</td>\n",
       "      <td>56.870058</td>\n",
       "      <td>161.372105</td>\n",
       "      <td>268.214249</td>\n",
       "      <td>0.990590</td>\n",
       "      <td>-0.598990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>solidity</th>\n",
       "      <td>6.0</td>\n",
       "      <td>0.623916</td>\n",
       "      <td>0.339846</td>\n",
       "      <td>0.152733</td>\n",
       "      <td>0.377325</td>\n",
       "      <td>0.679228</td>\n",
       "      <td>0.886518</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.404443</td>\n",
       "      <td>-1.620642</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     count        mean          std        min        25%  \\\n",
       "area                   6.0  172.166667   169.641288  13.000000  53.250000   \n",
       "convex_area            6.0  711.500000  1086.728071  13.000000  61.000000   \n",
       "eccentricity           6.0    0.656375     0.247219   0.329209   0.540171   \n",
       "equivalent_diameter    6.0   13.093467     7.571304   4.068429   8.223192   \n",
       "euler_number           6.0    1.000000     0.000000   1.000000   1.000000   \n",
       "extent                 6.0    0.447165     0.301338   0.082005   0.189305   \n",
       "filled_area            6.0  172.166667   169.641288  13.000000  53.250000   \n",
       "label                  6.0    3.666667     2.160247   1.000000   2.250000   \n",
       "major_axis_length      6.0   38.370325    44.316619   4.318667   8.900622   \n",
       "max_intensity          6.0    0.340745     0.124014   0.190913   0.284893   \n",
       "mean_intensity         6.0    0.219547     0.030212   0.179395   0.204378   \n",
       "min_intensity          6.0    0.146734     0.013942   0.134497   0.135899   \n",
       "minor_axis_length      6.0   15.843706    13.173532   3.679465   8.046784   \n",
       "orientation            6.0    0.481434     0.706937  -0.785398   0.359977   \n",
       "perimeter              6.0  100.797306   104.599710  10.242641  26.503048   \n",
       "solidity               6.0    0.623916     0.339846   0.152733   0.377325   \n",
       "\n",
       "                            50%         75%          max      skew  kurtosis  \n",
       "area                 103.500000  291.750000   422.000000  0.812830 -1.419060  \n",
       "convex_area          163.000000  895.750000  2763.000000  1.805071  2.969080  \n",
       "eccentricity           0.593026    0.846596     0.969315  0.263787 -1.189019  \n",
       "equivalent_diameter   11.248391   19.012782    23.179885  0.380733 -1.657971  \n",
       "euler_number           1.000000    1.000000     1.000000  0.000000  0.000000  \n",
       "extent                 0.488044    0.664015     0.812500 -0.203864 -1.892205  \n",
       "filled_area          103.500000  291.750000   422.000000  0.812830 -1.419060  \n",
       "label                  3.500000    4.750000     7.000000  0.462910 -0.300000  \n",
       "major_axis_length     14.965102   63.585279   109.301946  1.122066 -0.671105  \n",
       "max_intensity          0.334849    0.348291     0.562848  1.142450  2.509521  \n",
       "mean_intensity         0.212181    0.243441     0.257887  0.219149 -1.226143  \n",
       "min_intensity          0.143656    0.152434     0.169924  0.941806  0.040328  \n",
       "minor_axis_length     12.039514   18.343355    40.143232  1.533146  2.527297  \n",
       "orientation            0.544868    0.961182     1.185966 -1.251893  2.013372  \n",
       "perimeter             56.870058  161.372105   268.214249  0.990590 -0.598990  \n",
       "solidity               0.679228    0.886518     1.000000 -0.404443 -1.620642  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Running the pipeline for all images\n",
    "\n",
    "We have 20k images of this kind, some contain the lensing effect, other not.\n",
    "\n",
    "In this computer I have such data below `/home/chbrandt/fido/Data/GravLensChallenge/SpaceBasedTraining`.\n",
    "\n",
    "The dataset is composed by a *master table* where we find general properties for each *image* -- for instance,\n",
    "whether in that image there is a lens or not --, each image is referred by its *ID* number; `classifications.csv`\n",
    "is this table's filename.\n",
    "The image files are placed under `Data.0/Public/Band1/` and `Data.1/Public/Band1/` directories, they are *FITS*\n",
    "files named as `imageEUC_VIS-`*id*`.fits`.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([u'area', u'convex_area', u'eccentricity', u'equivalent_diameter',\n",
       "       u'euler_number', u'extent', u'filled_area', u'label',\n",
       "       u'major_axis_length', u'max_intensity', u'mean_intensity',\n",
       "       u'min_intensity', u'minor_axis_length', u'orientation', u'perimeter',\n",
       "       u'solidity'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_properties = features_matrix.index\n",
    "image_properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "tab_master_class = pandas.read_csv('classifications.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>is_lens</th>\n",
       "      <th>Einstein_area</th>\n",
       "      <th>numb_pix_lensed_image</th>\n",
       "      <th>flux_lensed_image_in_sigma</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100000</td>\n",
       "      <td>1</td>\n",
       "      <td>8.633760e-10</td>\n",
       "      <td>171</td>\n",
       "      <td>195.429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100001</td>\n",
       "      <td>1</td>\n",
       "      <td>1.317890e-10</td>\n",
       "      <td>294</td>\n",
       "      <td>855.589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100002</td>\n",
       "      <td>1</td>\n",
       "      <td>4.877250e-12</td>\n",
       "      <td>140</td>\n",
       "      <td>486.113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100003</td>\n",
       "      <td>1</td>\n",
       "      <td>1.440160e-09</td>\n",
       "      <td>1500</td>\n",
       "      <td>10467.400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100004</td>\n",
       "      <td>0</td>\n",
       "      <td>2.197350e-11</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19995</th>\n",
       "      <td>119995</td>\n",
       "      <td>0</td>\n",
       "      <td>1.041840e-09</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19996</th>\n",
       "      <td>119996</td>\n",
       "      <td>1</td>\n",
       "      <td>4.850260e-10</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19997</th>\n",
       "      <td>119997</td>\n",
       "      <td>0</td>\n",
       "      <td>1.395720e-11</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19998</th>\n",
       "      <td>119998</td>\n",
       "      <td>1</td>\n",
       "      <td>4.251510e-10</td>\n",
       "      <td>1024</td>\n",
       "      <td>4566.210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19999</th>\n",
       "      <td>119999</td>\n",
       "      <td>1</td>\n",
       "      <td>4.514050e-12</td>\n",
       "      <td>262</td>\n",
       "      <td>1741.430</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ID  is_lens  Einstein_area  numb_pix_lensed_image  \\\n",
       "0      100000        1   8.633760e-10                    171   \n",
       "1      100001        1   1.317890e-10                    294   \n",
       "2      100002        1   4.877250e-12                    140   \n",
       "3      100003        1   1.440160e-09                   1500   \n",
       "4      100004        0   2.197350e-11                      0   \n",
       "19995  119995        0   1.041840e-09                      0   \n",
       "19996  119996        1   4.850260e-10                      0   \n",
       "19997  119997        0   1.395720e-11                      0   \n",
       "19998  119998        1   4.251510e-10                   1024   \n",
       "19999  119999        1   4.514050e-12                    262   \n",
       "\n",
       "       flux_lensed_image_in_sigma  \n",
       "0                         195.429  \n",
       "1                         855.589  \n",
       "2                         486.113  \n",
       "3                       10467.400  \n",
       "4                           0.000  \n",
       "19995                       0.000  \n",
       "19996                       0.000  \n",
       "19997                       0.000  \n",
       "19998                    4566.210  \n",
       "19999                    1741.430  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pandas.concat([tab_master_class.head(), tab_master_class.tail()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chbrandt/.conda/envs/booq-dev/lib/python2.7/site-packages/ipykernel/__main__.py:151: DeprecationWarning: numpy boolean subtract, the `-` operator, is deprecated, use the bitwise_xor, the `^` operator, or the logical_xor function instead.\n",
      "/home/chbrandt/.conda/envs/booq-dev/lib/python2.7/site-packages/matplotlib/pyplot.py:524: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).\n",
      "  max_open_warning, RuntimeWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 58.1 s, sys: 387 ms, total: 58.5 s\n",
      "Wall time: 58.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "all_props = {}\n",
    "FAILED = []\n",
    "for i,row in enumerate(tab_master_class.iterrows()):\n",
    "    # for testing reasons:\n",
    "    if i == 100:\n",
    "        break\n",
    "\n",
    "    index,serie = row\n",
    "\n",
    "    id_column = 'ID'\n",
    "    class_column = 'is_lens'\n",
    "    \n",
    "    _id     = int(serie[id_column])\n",
    "    _isLens = bool(serie[class_column])\n",
    "    \n",
    "    _props = run_pipeline(_id,'datalinks/')\n",
    "    if _props is None:\n",
    "#         print \"Processing of ID:{} image/properties went wrong; Skipping this one.\".format(_id)\n",
    "        FAILED.append(_id)\n",
    "        continue\n",
    "        \n",
    "#     _props.fillna(-999,inplace=True)\n",
    "    _props[class_column] = int(_isLens)\n",
    "    del _isLens\n",
    "    \n",
    "    all_props[_id] = _props\n",
    "    del _id\n",
    "    del _props\n",
    "    \n",
    "import pandas\n",
    "data_panel = pandas.Panel(all_props)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<class 'pandas.core.panel.Panel'>\n",
       "Dimensions: 69 (items) x 16 (major_axis) x 11 (minor_axis)\n",
       "Items axis: 100000 to 100099\n",
       "Major_axis axis: area to solidity\n",
       "Minor_axis axis: count to is_lens"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_panel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dpT = data_panel.transpose(1,0,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([u'area', u'convex_area', u'eccentricity', u'equivalent_diameter',\n",
       "       u'euler_number', u'extent', u'filled_area', u'label',\n",
       "       u'major_axis_length', u'max_intensity', u'mean_intensity',\n",
       "       u'min_intensity', u'minor_axis_length', u'orientation', u'perimeter',\n",
       "       u'solidity'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dpT.items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "      <th>skew</th>\n",
       "      <th>kurtosis</th>\n",
       "      <th>is_lens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100000</th>\n",
       "      <td>2.0</td>\n",
       "      <td>86.000000</td>\n",
       "      <td>98.994949</td>\n",
       "      <td>16.0</td>\n",
       "      <td>51.00</td>\n",
       "      <td>86.0</td>\n",
       "      <td>121.00</td>\n",
       "      <td>156.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100001</th>\n",
       "      <td>1.0</td>\n",
       "      <td>589.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>589.0</td>\n",
       "      <td>589.00</td>\n",
       "      <td>589.0</td>\n",
       "      <td>589.00</td>\n",
       "      <td>589.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100002</th>\n",
       "      <td>1.0</td>\n",
       "      <td>171.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>171.0</td>\n",
       "      <td>171.00</td>\n",
       "      <td>171.0</td>\n",
       "      <td>171.00</td>\n",
       "      <td>171.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100003</th>\n",
       "      <td>6.0</td>\n",
       "      <td>172.166667</td>\n",
       "      <td>169.641288</td>\n",
       "      <td>13.0</td>\n",
       "      <td>53.25</td>\n",
       "      <td>103.5</td>\n",
       "      <td>291.75</td>\n",
       "      <td>422.0</td>\n",
       "      <td>0.812830</td>\n",
       "      <td>-1.419060</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100004</th>\n",
       "      <td>2.0</td>\n",
       "      <td>134.000000</td>\n",
       "      <td>134.350288</td>\n",
       "      <td>39.0</td>\n",
       "      <td>86.50</td>\n",
       "      <td>134.0</td>\n",
       "      <td>181.50</td>\n",
       "      <td>229.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100007</th>\n",
       "      <td>2.0</td>\n",
       "      <td>258.000000</td>\n",
       "      <td>336.582828</td>\n",
       "      <td>20.0</td>\n",
       "      <td>139.00</td>\n",
       "      <td>258.0</td>\n",
       "      <td>377.00</td>\n",
       "      <td>496.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100008</th>\n",
       "      <td>1.0</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.0</td>\n",
       "      <td>11.00</td>\n",
       "      <td>11.0</td>\n",
       "      <td>11.00</td>\n",
       "      <td>11.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100009</th>\n",
       "      <td>2.0</td>\n",
       "      <td>143.500000</td>\n",
       "      <td>161.927453</td>\n",
       "      <td>29.0</td>\n",
       "      <td>86.25</td>\n",
       "      <td>143.5</td>\n",
       "      <td>200.75</td>\n",
       "      <td>258.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100010</th>\n",
       "      <td>1.0</td>\n",
       "      <td>238.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>238.0</td>\n",
       "      <td>238.00</td>\n",
       "      <td>238.0</td>\n",
       "      <td>238.00</td>\n",
       "      <td>238.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100011</th>\n",
       "      <td>3.0</td>\n",
       "      <td>59.000000</td>\n",
       "      <td>46.227697</td>\n",
       "      <td>6.0</td>\n",
       "      <td>43.00</td>\n",
       "      <td>80.0</td>\n",
       "      <td>85.50</td>\n",
       "      <td>91.0</td>\n",
       "      <td>-1.622374</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100012</th>\n",
       "      <td>3.0</td>\n",
       "      <td>27.666667</td>\n",
       "      <td>17.559423</td>\n",
       "      <td>11.0</td>\n",
       "      <td>18.50</td>\n",
       "      <td>26.0</td>\n",
       "      <td>36.00</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0.423273</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100013</th>\n",
       "      <td>1.0</td>\n",
       "      <td>128.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>128.0</td>\n",
       "      <td>128.00</td>\n",
       "      <td>128.0</td>\n",
       "      <td>128.00</td>\n",
       "      <td>128.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100014</th>\n",
       "      <td>1.0</td>\n",
       "      <td>535.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>535.0</td>\n",
       "      <td>535.00</td>\n",
       "      <td>535.0</td>\n",
       "      <td>535.00</td>\n",
       "      <td>535.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100018</th>\n",
       "      <td>1.0</td>\n",
       "      <td>260.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>260.0</td>\n",
       "      <td>260.00</td>\n",
       "      <td>260.0</td>\n",
       "      <td>260.00</td>\n",
       "      <td>260.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100019</th>\n",
       "      <td>5.0</td>\n",
       "      <td>172.400000</td>\n",
       "      <td>28.771514</td>\n",
       "      <td>141.0</td>\n",
       "      <td>142.00</td>\n",
       "      <td>185.0</td>\n",
       "      <td>193.00</td>\n",
       "      <td>201.0</td>\n",
       "      <td>-0.435923</td>\n",
       "      <td>-3.073837</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100020</th>\n",
       "      <td>5.0</td>\n",
       "      <td>152.000000</td>\n",
       "      <td>261.561656</td>\n",
       "      <td>19.0</td>\n",
       "      <td>21.00</td>\n",
       "      <td>26.0</td>\n",
       "      <td>76.00</td>\n",
       "      <td>618.0</td>\n",
       "      <td>2.192341</td>\n",
       "      <td>4.832253</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100021</th>\n",
       "      <td>1.0</td>\n",
       "      <td>39.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>39.0</td>\n",
       "      <td>39.00</td>\n",
       "      <td>39.0</td>\n",
       "      <td>39.00</td>\n",
       "      <td>39.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100022</th>\n",
       "      <td>3.0</td>\n",
       "      <td>102.333333</td>\n",
       "      <td>110.065132</td>\n",
       "      <td>30.0</td>\n",
       "      <td>39.00</td>\n",
       "      <td>48.0</td>\n",
       "      <td>138.50</td>\n",
       "      <td>229.0</td>\n",
       "      <td>1.680082</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100024</th>\n",
       "      <td>2.0</td>\n",
       "      <td>88.500000</td>\n",
       "      <td>26.162951</td>\n",
       "      <td>70.0</td>\n",
       "      <td>79.25</td>\n",
       "      <td>88.5</td>\n",
       "      <td>97.75</td>\n",
       "      <td>107.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100026</th>\n",
       "      <td>2.0</td>\n",
       "      <td>191.000000</td>\n",
       "      <td>226.274170</td>\n",
       "      <td>31.0</td>\n",
       "      <td>111.00</td>\n",
       "      <td>191.0</td>\n",
       "      <td>271.00</td>\n",
       "      <td>351.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100027</th>\n",
       "      <td>2.0</td>\n",
       "      <td>96.500000</td>\n",
       "      <td>53.033009</td>\n",
       "      <td>59.0</td>\n",
       "      <td>77.75</td>\n",
       "      <td>96.5</td>\n",
       "      <td>115.25</td>\n",
       "      <td>134.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100028</th>\n",
       "      <td>1.0</td>\n",
       "      <td>441.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>441.0</td>\n",
       "      <td>441.00</td>\n",
       "      <td>441.0</td>\n",
       "      <td>441.00</td>\n",
       "      <td>441.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100029</th>\n",
       "      <td>5.0</td>\n",
       "      <td>137.200000</td>\n",
       "      <td>109.207143</td>\n",
       "      <td>25.0</td>\n",
       "      <td>33.00</td>\n",
       "      <td>166.0</td>\n",
       "      <td>177.00</td>\n",
       "      <td>285.0</td>\n",
       "      <td>0.246900</td>\n",
       "      <td>-1.349339</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100031</th>\n",
       "      <td>3.0</td>\n",
       "      <td>388.000000</td>\n",
       "      <td>474.667252</td>\n",
       "      <td>105.0</td>\n",
       "      <td>114.00</td>\n",
       "      <td>123.0</td>\n",
       "      <td>529.50</td>\n",
       "      <td>936.0</td>\n",
       "      <td>1.729249</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100032</th>\n",
       "      <td>3.0</td>\n",
       "      <td>152.000000</td>\n",
       "      <td>121.577136</td>\n",
       "      <td>73.0</td>\n",
       "      <td>82.00</td>\n",
       "      <td>91.0</td>\n",
       "      <td>191.50</td>\n",
       "      <td>292.0</td>\n",
       "      <td>1.689436</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100033</th>\n",
       "      <td>1.0</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80.0</td>\n",
       "      <td>80.00</td>\n",
       "      <td>80.0</td>\n",
       "      <td>80.00</td>\n",
       "      <td>80.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100034</th>\n",
       "      <td>1.0</td>\n",
       "      <td>367.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>367.0</td>\n",
       "      <td>367.00</td>\n",
       "      <td>367.0</td>\n",
       "      <td>367.00</td>\n",
       "      <td>367.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100035</th>\n",
       "      <td>2.0</td>\n",
       "      <td>61.500000</td>\n",
       "      <td>24.748737</td>\n",
       "      <td>44.0</td>\n",
       "      <td>52.75</td>\n",
       "      <td>61.5</td>\n",
       "      <td>70.25</td>\n",
       "      <td>79.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100036</th>\n",
       "      <td>3.0</td>\n",
       "      <td>128.000000</td>\n",
       "      <td>158.521292</td>\n",
       "      <td>33.0</td>\n",
       "      <td>36.50</td>\n",
       "      <td>40.0</td>\n",
       "      <td>175.50</td>\n",
       "      <td>311.0</td>\n",
       "      <td>1.728252</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100038</th>\n",
       "      <td>1.0</td>\n",
       "      <td>180.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>180.0</td>\n",
       "      <td>180.00</td>\n",
       "      <td>180.0</td>\n",
       "      <td>180.00</td>\n",
       "      <td>180.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100051</th>\n",
       "      <td>5.0</td>\n",
       "      <td>147.400000</td>\n",
       "      <td>69.783952</td>\n",
       "      <td>60.0</td>\n",
       "      <td>84.00</td>\n",
       "      <td>189.0</td>\n",
       "      <td>194.00</td>\n",
       "      <td>210.0</td>\n",
       "      <td>-0.617909</td>\n",
       "      <td>-2.856899</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100052</th>\n",
       "      <td>4.0</td>\n",
       "      <td>114.250000</td>\n",
       "      <td>75.066082</td>\n",
       "      <td>22.0</td>\n",
       "      <td>70.75</td>\n",
       "      <td>122.5</td>\n",
       "      <td>166.00</td>\n",
       "      <td>190.0</td>\n",
       "      <td>-0.452160</td>\n",
       "      <td>-1.998302</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100053</th>\n",
       "      <td>2.0</td>\n",
       "      <td>163.000000</td>\n",
       "      <td>171.119841</td>\n",
       "      <td>42.0</td>\n",
       "      <td>102.50</td>\n",
       "      <td>163.0</td>\n",
       "      <td>223.50</td>\n",
       "      <td>284.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100054</th>\n",
       "      <td>1.0</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21.0</td>\n",
       "      <td>21.00</td>\n",
       "      <td>21.0</td>\n",
       "      <td>21.00</td>\n",
       "      <td>21.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100056</th>\n",
       "      <td>1.0</td>\n",
       "      <td>88.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>88.0</td>\n",
       "      <td>88.00</td>\n",
       "      <td>88.0</td>\n",
       "      <td>88.00</td>\n",
       "      <td>88.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100057</th>\n",
       "      <td>4.0</td>\n",
       "      <td>134.000000</td>\n",
       "      <td>110.751975</td>\n",
       "      <td>34.0</td>\n",
       "      <td>41.50</td>\n",
       "      <td>127.5</td>\n",
       "      <td>220.00</td>\n",
       "      <td>247.0</td>\n",
       "      <td>0.083637</td>\n",
       "      <td>-5.439761</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100058</th>\n",
       "      <td>3.0</td>\n",
       "      <td>116.000000</td>\n",
       "      <td>45.133136</td>\n",
       "      <td>73.0</td>\n",
       "      <td>92.50</td>\n",
       "      <td>112.0</td>\n",
       "      <td>137.50</td>\n",
       "      <td>163.0</td>\n",
       "      <td>0.395687</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100059</th>\n",
       "      <td>1.0</td>\n",
       "      <td>78.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>78.0</td>\n",
       "      <td>78.00</td>\n",
       "      <td>78.0</td>\n",
       "      <td>78.00</td>\n",
       "      <td>78.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100060</th>\n",
       "      <td>3.0</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>114.354711</td>\n",
       "      <td>30.0</td>\n",
       "      <td>33.00</td>\n",
       "      <td>36.0</td>\n",
       "      <td>133.50</td>\n",
       "      <td>231.0</td>\n",
       "      <td>1.726688</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100061</th>\n",
       "      <td>2.0</td>\n",
       "      <td>133.000000</td>\n",
       "      <td>76.367532</td>\n",
       "      <td>79.0</td>\n",
       "      <td>106.00</td>\n",
       "      <td>133.0</td>\n",
       "      <td>160.00</td>\n",
       "      <td>187.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100063</th>\n",
       "      <td>2.0</td>\n",
       "      <td>119.000000</td>\n",
       "      <td>93.338095</td>\n",
       "      <td>53.0</td>\n",
       "      <td>86.00</td>\n",
       "      <td>119.0</td>\n",
       "      <td>152.00</td>\n",
       "      <td>185.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100064</th>\n",
       "      <td>4.0</td>\n",
       "      <td>101.250000</td>\n",
       "      <td>110.593475</td>\n",
       "      <td>40.0</td>\n",
       "      <td>45.25</td>\n",
       "      <td>49.0</td>\n",
       "      <td>105.00</td>\n",
       "      <td>267.0</td>\n",
       "      <td>1.989824</td>\n",
       "      <td>3.966699</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100065</th>\n",
       "      <td>4.0</td>\n",
       "      <td>94.500000</td>\n",
       "      <td>78.053401</td>\n",
       "      <td>38.0</td>\n",
       "      <td>46.25</td>\n",
       "      <td>66.0</td>\n",
       "      <td>114.25</td>\n",
       "      <td>208.0</td>\n",
       "      <td>1.662794</td>\n",
       "      <td>2.705383</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100069</th>\n",
       "      <td>1.0</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17.0</td>\n",
       "      <td>17.00</td>\n",
       "      <td>17.0</td>\n",
       "      <td>17.00</td>\n",
       "      <td>17.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100070</th>\n",
       "      <td>3.0</td>\n",
       "      <td>71.666667</td>\n",
       "      <td>51.733290</td>\n",
       "      <td>23.0</td>\n",
       "      <td>44.50</td>\n",
       "      <td>66.0</td>\n",
       "      <td>96.00</td>\n",
       "      <td>126.0</td>\n",
       "      <td>0.486999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100072</th>\n",
       "      <td>4.0</td>\n",
       "      <td>115.250000</td>\n",
       "      <td>107.145929</td>\n",
       "      <td>29.0</td>\n",
       "      <td>40.25</td>\n",
       "      <td>84.5</td>\n",
       "      <td>159.50</td>\n",
       "      <td>263.0</td>\n",
       "      <td>1.204814</td>\n",
       "      <td>0.604404</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100073</th>\n",
       "      <td>1.0</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>26.0</td>\n",
       "      <td>26.00</td>\n",
       "      <td>26.0</td>\n",
       "      <td>26.00</td>\n",
       "      <td>26.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100078</th>\n",
       "      <td>2.0</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>1.414214</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.50</td>\n",
       "      <td>13.0</td>\n",
       "      <td>13.50</td>\n",
       "      <td>14.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100079</th>\n",
       "      <td>3.0</td>\n",
       "      <td>107.666667</td>\n",
       "      <td>54.077105</td>\n",
       "      <td>52.0</td>\n",
       "      <td>81.50</td>\n",
       "      <td>111.0</td>\n",
       "      <td>135.50</td>\n",
       "      <td>160.0</td>\n",
       "      <td>-0.276328</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100082</th>\n",
       "      <td>1.0</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>36.0</td>\n",
       "      <td>36.00</td>\n",
       "      <td>36.0</td>\n",
       "      <td>36.00</td>\n",
       "      <td>36.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100084</th>\n",
       "      <td>3.0</td>\n",
       "      <td>194.666667</td>\n",
       "      <td>190.148188</td>\n",
       "      <td>54.0</td>\n",
       "      <td>86.50</td>\n",
       "      <td>119.0</td>\n",
       "      <td>265.00</td>\n",
       "      <td>411.0</td>\n",
       "      <td>1.507146</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100086</th>\n",
       "      <td>1.0</td>\n",
       "      <td>257.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>257.0</td>\n",
       "      <td>257.00</td>\n",
       "      <td>257.0</td>\n",
       "      <td>257.00</td>\n",
       "      <td>257.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100087</th>\n",
       "      <td>3.0</td>\n",
       "      <td>131.333333</td>\n",
       "      <td>182.979052</td>\n",
       "      <td>12.0</td>\n",
       "      <td>26.00</td>\n",
       "      <td>40.0</td>\n",
       "      <td>191.00</td>\n",
       "      <td>342.0</td>\n",
       "      <td>1.686535</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100088</th>\n",
       "      <td>1.0</td>\n",
       "      <td>159.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>159.0</td>\n",
       "      <td>159.00</td>\n",
       "      <td>159.0</td>\n",
       "      <td>159.00</td>\n",
       "      <td>159.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100093</th>\n",
       "      <td>3.0</td>\n",
       "      <td>139.666667</td>\n",
       "      <td>164.894916</td>\n",
       "      <td>40.0</td>\n",
       "      <td>44.50</td>\n",
       "      <td>49.0</td>\n",
       "      <td>189.50</td>\n",
       "      <td>330.0</td>\n",
       "      <td>1.726248</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100095</th>\n",
       "      <td>2.0</td>\n",
       "      <td>23.500000</td>\n",
       "      <td>9.192388</td>\n",
       "      <td>17.0</td>\n",
       "      <td>20.25</td>\n",
       "      <td>23.5</td>\n",
       "      <td>26.75</td>\n",
       "      <td>30.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100096</th>\n",
       "      <td>2.0</td>\n",
       "      <td>20.500000</td>\n",
       "      <td>10.606602</td>\n",
       "      <td>13.0</td>\n",
       "      <td>16.75</td>\n",
       "      <td>20.5</td>\n",
       "      <td>24.25</td>\n",
       "      <td>28.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100097</th>\n",
       "      <td>1.0</td>\n",
       "      <td>204.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>204.0</td>\n",
       "      <td>204.00</td>\n",
       "      <td>204.0</td>\n",
       "      <td>204.00</td>\n",
       "      <td>204.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100098</th>\n",
       "      <td>3.0</td>\n",
       "      <td>119.333333</td>\n",
       "      <td>49.500842</td>\n",
       "      <td>70.0</td>\n",
       "      <td>94.50</td>\n",
       "      <td>119.0</td>\n",
       "      <td>144.00</td>\n",
       "      <td>169.0</td>\n",
       "      <td>0.030301</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100099</th>\n",
       "      <td>2.0</td>\n",
       "      <td>291.500000</td>\n",
       "      <td>379.716341</td>\n",
       "      <td>23.0</td>\n",
       "      <td>157.25</td>\n",
       "      <td>291.5</td>\n",
       "      <td>425.75</td>\n",
       "      <td>560.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>69 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        count        mean         std    min     25%    50%     75%    max  \\\n",
       "100000    2.0   86.000000   98.994949   16.0   51.00   86.0  121.00  156.0   \n",
       "100001    1.0  589.000000         NaN  589.0  589.00  589.0  589.00  589.0   \n",
       "100002    1.0  171.000000         NaN  171.0  171.00  171.0  171.00  171.0   \n",
       "100003    6.0  172.166667  169.641288   13.0   53.25  103.5  291.75  422.0   \n",
       "100004    2.0  134.000000  134.350288   39.0   86.50  134.0  181.50  229.0   \n",
       "100007    2.0  258.000000  336.582828   20.0  139.00  258.0  377.00  496.0   \n",
       "100008    1.0   11.000000         NaN   11.0   11.00   11.0   11.00   11.0   \n",
       "100009    2.0  143.500000  161.927453   29.0   86.25  143.5  200.75  258.0   \n",
       "100010    1.0  238.000000         NaN  238.0  238.00  238.0  238.00  238.0   \n",
       "100011    3.0   59.000000   46.227697    6.0   43.00   80.0   85.50   91.0   \n",
       "100012    3.0   27.666667   17.559423   11.0   18.50   26.0   36.00   46.0   \n",
       "100013    1.0  128.000000         NaN  128.0  128.00  128.0  128.00  128.0   \n",
       "100014    1.0  535.000000         NaN  535.0  535.00  535.0  535.00  535.0   \n",
       "100018    1.0  260.000000         NaN  260.0  260.00  260.0  260.00  260.0   \n",
       "100019    5.0  172.400000   28.771514  141.0  142.00  185.0  193.00  201.0   \n",
       "100020    5.0  152.000000  261.561656   19.0   21.00   26.0   76.00  618.0   \n",
       "100021    1.0   39.000000         NaN   39.0   39.00   39.0   39.00   39.0   \n",
       "100022    3.0  102.333333  110.065132   30.0   39.00   48.0  138.50  229.0   \n",
       "100024    2.0   88.500000   26.162951   70.0   79.25   88.5   97.75  107.0   \n",
       "100026    2.0  191.000000  226.274170   31.0  111.00  191.0  271.00  351.0   \n",
       "100027    2.0   96.500000   53.033009   59.0   77.75   96.5  115.25  134.0   \n",
       "100028    1.0  441.000000         NaN  441.0  441.00  441.0  441.00  441.0   \n",
       "100029    5.0  137.200000  109.207143   25.0   33.00  166.0  177.00  285.0   \n",
       "100031    3.0  388.000000  474.667252  105.0  114.00  123.0  529.50  936.0   \n",
       "100032    3.0  152.000000  121.577136   73.0   82.00   91.0  191.50  292.0   \n",
       "100033    1.0   80.000000         NaN   80.0   80.00   80.0   80.00   80.0   \n",
       "100034    1.0  367.000000         NaN  367.0  367.00  367.0  367.00  367.0   \n",
       "100035    2.0   61.500000   24.748737   44.0   52.75   61.5   70.25   79.0   \n",
       "100036    3.0  128.000000  158.521292   33.0   36.50   40.0  175.50  311.0   \n",
       "100038    1.0  180.000000         NaN  180.0  180.00  180.0  180.00  180.0   \n",
       "...       ...         ...         ...    ...     ...    ...     ...    ...   \n",
       "100051    5.0  147.400000   69.783952   60.0   84.00  189.0  194.00  210.0   \n",
       "100052    4.0  114.250000   75.066082   22.0   70.75  122.5  166.00  190.0   \n",
       "100053    2.0  163.000000  171.119841   42.0  102.50  163.0  223.50  284.0   \n",
       "100054    1.0   21.000000         NaN   21.0   21.00   21.0   21.00   21.0   \n",
       "100056    1.0   88.000000         NaN   88.0   88.00   88.0   88.00   88.0   \n",
       "100057    4.0  134.000000  110.751975   34.0   41.50  127.5  220.00  247.0   \n",
       "100058    3.0  116.000000   45.133136   73.0   92.50  112.0  137.50  163.0   \n",
       "100059    1.0   78.000000         NaN   78.0   78.00   78.0   78.00   78.0   \n",
       "100060    3.0   99.000000  114.354711   30.0   33.00   36.0  133.50  231.0   \n",
       "100061    2.0  133.000000   76.367532   79.0  106.00  133.0  160.00  187.0   \n",
       "100063    2.0  119.000000   93.338095   53.0   86.00  119.0  152.00  185.0   \n",
       "100064    4.0  101.250000  110.593475   40.0   45.25   49.0  105.00  267.0   \n",
       "100065    4.0   94.500000   78.053401   38.0   46.25   66.0  114.25  208.0   \n",
       "100069    1.0   17.000000         NaN   17.0   17.00   17.0   17.00   17.0   \n",
       "100070    3.0   71.666667   51.733290   23.0   44.50   66.0   96.00  126.0   \n",
       "100072    4.0  115.250000  107.145929   29.0   40.25   84.5  159.50  263.0   \n",
       "100073    1.0   26.000000         NaN   26.0   26.00   26.0   26.00   26.0   \n",
       "100078    2.0   13.000000    1.414214   12.0   12.50   13.0   13.50   14.0   \n",
       "100079    3.0  107.666667   54.077105   52.0   81.50  111.0  135.50  160.0   \n",
       "100082    1.0   36.000000         NaN   36.0   36.00   36.0   36.00   36.0   \n",
       "100084    3.0  194.666667  190.148188   54.0   86.50  119.0  265.00  411.0   \n",
       "100086    1.0  257.000000         NaN  257.0  257.00  257.0  257.00  257.0   \n",
       "100087    3.0  131.333333  182.979052   12.0   26.00   40.0  191.00  342.0   \n",
       "100088    1.0  159.000000         NaN  159.0  159.00  159.0  159.00  159.0   \n",
       "100093    3.0  139.666667  164.894916   40.0   44.50   49.0  189.50  330.0   \n",
       "100095    2.0   23.500000    9.192388   17.0   20.25   23.5   26.75   30.0   \n",
       "100096    2.0   20.500000   10.606602   13.0   16.75   20.5   24.25   28.0   \n",
       "100097    1.0  204.000000         NaN  204.0  204.00  204.0  204.00  204.0   \n",
       "100098    3.0  119.333333   49.500842   70.0   94.50  119.0  144.00  169.0   \n",
       "100099    2.0  291.500000  379.716341   23.0  157.25  291.5  425.75  560.0   \n",
       "\n",
       "            skew  kurtosis  is_lens  \n",
       "100000       NaN       NaN      1.0  \n",
       "100001       NaN       NaN      1.0  \n",
       "100002       NaN       NaN      1.0  \n",
       "100003  0.812830 -1.419060      1.0  \n",
       "100004       NaN       NaN      0.0  \n",
       "100007       NaN       NaN      1.0  \n",
       "100008       NaN       NaN      1.0  \n",
       "100009       NaN       NaN      0.0  \n",
       "100010       NaN       NaN      1.0  \n",
       "100011 -1.622374       NaN      1.0  \n",
       "100012  0.423273       NaN      0.0  \n",
       "100013       NaN       NaN      1.0  \n",
       "100014       NaN       NaN      1.0  \n",
       "100018       NaN       NaN      1.0  \n",
       "100019 -0.435923 -3.073837      1.0  \n",
       "100020  2.192341  4.832253      1.0  \n",
       "100021       NaN       NaN      1.0  \n",
       "100022  1.680082       NaN      1.0  \n",
       "100024       NaN       NaN      1.0  \n",
       "100026       NaN       NaN      0.0  \n",
       "100027       NaN       NaN      1.0  \n",
       "100028       NaN       NaN      0.0  \n",
       "100029  0.246900 -1.349339      1.0  \n",
       "100031  1.729249       NaN      1.0  \n",
       "100032  1.689436       NaN      1.0  \n",
       "100033       NaN       NaN      1.0  \n",
       "100034       NaN       NaN      1.0  \n",
       "100035       NaN       NaN      1.0  \n",
       "100036  1.728252       NaN      1.0  \n",
       "100038       NaN       NaN      1.0  \n",
       "...          ...       ...      ...  \n",
       "100051 -0.617909 -2.856899      1.0  \n",
       "100052 -0.452160 -1.998302      1.0  \n",
       "100053       NaN       NaN      0.0  \n",
       "100054       NaN       NaN      1.0  \n",
       "100056       NaN       NaN      1.0  \n",
       "100057  0.083637 -5.439761      1.0  \n",
       "100058  0.395687       NaN      0.0  \n",
       "100059       NaN       NaN      0.0  \n",
       "100060  1.726688       NaN      1.0  \n",
       "100061       NaN       NaN      1.0  \n",
       "100063       NaN       NaN      0.0  \n",
       "100064  1.989824  3.966699      1.0  \n",
       "100065  1.662794  2.705383      1.0  \n",
       "100069       NaN       NaN      0.0  \n",
       "100070  0.486999       NaN      1.0  \n",
       "100072  1.204814  0.604404      1.0  \n",
       "100073       NaN       NaN      1.0  \n",
       "100078       NaN       NaN      1.0  \n",
       "100079 -0.276328       NaN      0.0  \n",
       "100082       NaN       NaN      0.0  \n",
       "100084  1.507146       NaN      0.0  \n",
       "100086       NaN       NaN      1.0  \n",
       "100087  1.686535       NaN      1.0  \n",
       "100088       NaN       NaN      0.0  \n",
       "100093  1.726248       NaN      1.0  \n",
       "100095       NaN       NaN      0.0  \n",
       "100096       NaN       NaN      0.0  \n",
       "100097       NaN       NaN      1.0  \n",
       "100098  0.030301       NaN      1.0  \n",
       "100099       NaN       NaN      1.0  \n",
       "\n",
       "[69 rows x 11 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dpT['area']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "31"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(FAILED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[100005,\n",
       " 100006,\n",
       " 100015,\n",
       " 100016,\n",
       " 100017,\n",
       " 100023,\n",
       " 100025,\n",
       " 100030,\n",
       " 100037,\n",
       " 100041,\n",
       " 100044,\n",
       " 100047,\n",
       " 100055,\n",
       " 100062,\n",
       " 100066,\n",
       " 100067,\n",
       " 100068,\n",
       " 100071,\n",
       " 100074,\n",
       " 100075,\n",
       " 100076,\n",
       " 100077,\n",
       " 100080,\n",
       " 100081,\n",
       " 100083,\n",
       " 100085,\n",
       " 100089,\n",
       " 100090,\n",
       " 100091,\n",
       " 100092,\n",
       " 100094]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FAILED"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:booq-dev]",
   "language": "python",
   "name": "conda-env-booq-dev-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}