REDD_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "No such file as data/redd.h5",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 7\u001b[0m\n\u001b[1;32m      5\u001b[0m plt\u001b[38;5;241m.\u001b[39mstyle\u001b[38;5;241m.\u001b[39muse(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mggplot\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mnilmtk\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m DataSet\n\u001b[0;32m----> 7\u001b[0m redd \u001b[38;5;241m=\u001b[39m \u001b[43mDataSet\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mdata/redd.h5\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      8\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpickle\u001b[39;00m\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/nilmtk-env/lib/python3.8/site-packages/nilmtk/dataset.py:45\u001b[0m, in \u001b[0;36mDataSet.__init__\u001b[0;34m(self, filename, format)\u001b[0m\n\u001b[1;32m     43\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata \u001b[38;5;241m=\u001b[39m {}\n\u001b[1;32m     44\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m filename \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m---> 45\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mimport_metadata(\u001b[43mget_datastore\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[43m)\u001b[49m)\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/nilmtk-env/lib/python3.8/site-packages/nilmtk/utils.py:323\u001b[0m, in \u001b[0;36mget_datastore\u001b[0;34m(filename, format, mode)\u001b[0m\n\u001b[1;32m    321\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m filename \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    322\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mformat\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mHDF\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 323\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mHDFDataStore\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    324\u001b[0m     \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mformat\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCSV\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m    325\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m CSVDataStore(filename)\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/nilmtk-env/lib/python3.8/site-packages/nilmtk/docinherit.py:53\u001b[0m, in \u001b[0;36mDocInherit.__get__.<locals>.f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     50\u001b[0m \u001b[38;5;129m@wraps\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmthd, assigned\u001b[38;5;241m=\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__name__\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__module__\u001b[39m\u001b[38;5;124m'\u001b[39m))\n\u001b[1;32m     51\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mf\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     52\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m obj:\n\u001b[0;32m---> 53\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmthd\u001b[49m\u001b[43m(\u001b[49m\u001b[43mobj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     54\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     55\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmthd(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/nilmtk-env/lib/python3.8/site-packages/nilmtk/datastore/hdfdatastore.py:18\u001b[0m, in \u001b[0;36mHDFDataStore.__init__\u001b[0;34m(self, filename, mode)\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[38;5;129m@doc_inherit\u001b[39m\n\u001b[1;32m     16\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, filename, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[1;32m     17\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m mode \u001b[38;5;129;01min\u001b[39;00m [ \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124ma\u001b[39m\u001b[38;5;124m\"\u001b[39m ] \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m isfile(filename):\n\u001b[0;32m---> 18\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mIOError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNo such file as \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m filename)\n\u001b[1;32m     20\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m warnings\u001b[38;5;241m.\u001b[39mcatch_warnings():\n\u001b[1;32m     21\u001b[0m         \u001b[38;5;66;03m# Silence pytables warnings with numpy, out of our control\u001b[39;00m\n\u001b[1;32m     22\u001b[0m         warnings\u001b[38;5;241m.\u001b[39mfilterwarnings(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mignore\u001b[39m\u001b[38;5;124m'\u001b[39m, category\u001b[38;5;241m=\u001b[39m\u001b[38;5;167;01mRuntimeWarning\u001b[39;00m, message\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.*numpy.ufunc size changed.*\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "\u001b[0;31mOSError\u001b[0m: No such file as data/redd.h5"
     ]
    }
   ],
   "source": [
    "from matplotlib import rcParams\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "rcParams['figure.figsize'] = (13, 6)\n",
    "plt.style.use('ggplot')\n",
    "from nilmtk import DataSet\n",
    "redd = DataSet('data/redd.h5')\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Extracting long continuous data\n",
    "\n",
    "- app_time_list: Time stamp data in appliances.\n",
    "- main_time_list: Time stamp data in main.\n",
    "- index: longest continuous data range index as appliances."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.1. Time extracting from main and app"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "app_time_list = []\n",
    "for j in range(len(redd.buildings)):\n",
    "    elec = redd.buildings[j+1].elec\n",
    "    df = elec['dish washer'].load().next()\n",
    "    \n",
    "    t_list = []\n",
    "    for i in range(len(df)):\n",
    "        a = df.index[i]\n",
    "        date = a.ctime().split(' ')\n",
    "        if '' in date:\n",
    "            date.remove('')\n",
    "        time = date[3].split(':')\n",
    "        t = int(date[2]) * 86400 + int(time[0]) * 3600 + int(time[1]) * 60 + int(time[2])\n",
    "        if date[1] == 'May':\n",
    "            t += 30 * 86400\n",
    "        if date[1] == 'Jun':\n",
    "            t += 61 * 86400\n",
    "        t_list.append(t)\n",
    "        \n",
    "        if i%10000 == 0:\n",
    "            print i\n",
    "    app_time_list.append(t_list)\n",
    "    \n",
    "main_time_list = []\n",
    "for j in range(len(redd.buildings)):\n",
    "    elec = redd.buildings[j+1].elec\n",
    "    df = elec.mains().load().next()\n",
    "    \n",
    "    t_list = []\n",
    "    for i in range(len(df)):\n",
    "        a = df.index[i]\n",
    "        date = a.ctime().split(' ')\n",
    "        if '' in date:\n",
    "            date.remove('')\n",
    "        time = date[3].split(':')\n",
    "        t = int(date[2]) * 86400 + int(time[0]) * 3600 + int(time[1]) * 60 + int(time[2])\n",
    "        if date[1] == 'May':\n",
    "            t += 30 * 86400\n",
    "        if date[1] == 'Jun':\n",
    "            t += 61 * 86400\n",
    "        t_list.append(t)\n",
    "        \n",
    "        if i%10000 == 0:\n",
    "            print i\n",
    "    main_time_list.append(t_list)\n",
    "    \n",
    "diffreq_app_time_list = []\n",
    "diffreq_index_list = [[1, 'washer dryer'], [3, 'washer dryer'], [5, 'washer dryer'], [5, 'electric space heater']]\n",
    "for diffreq in diffreq_index_list:\n",
    "    houseno = diffreq[0]\n",
    "    appname = diffreq[1]\n",
    "    \n",
    "    elec = redd.buildings[houseno].elec\n",
    "    df = elec[appname].load().next()\n",
    "    \n",
    "    t_list = []\n",
    "    for i in range(len(df)):\n",
    "        a = df.index[i]\n",
    "        date = a.ctime().split(' ')\n",
    "        if '' in date:\n",
    "            date.remove('')\n",
    "        time = date[3].split(':')\n",
    "        t = int(date[2]) * 86400 + int(time[0]) * 3600 + int(time[1]) * 60 + int(time[2])\n",
    "        if date[1] == 'May':\n",
    "            t += 30 * 86400\n",
    "        if date[1] == 'Jun':\n",
    "            t += 61 * 86400\n",
    "        t_list.append(t)\n",
    "        \n",
    "        if i%10000 == 0:\n",
    "            print i\n",
    "    diffreq_app_time_list.append(t_list)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "import pickle\n",
    "\n",
    "with open('redd_app_time', 'wb') as fp:\n",
    "    pickle.dump(app_time_list, fp)\n",
    "    \n",
    "with open('redd_main_time', 'wb') as fp:\n",
    "    pickle.dump(main_time_list, fp)\n",
    "    \n",
    "with open('redd_diffreq_time', 'wb') as fp:\n",
    "    pickle.dump(diffreq_app_time_list, fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "you can easily load app_time_list and main_time_list from pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open ('redd_app_time', 'rb') as fp:\n",
    "    app_time_list = pickle.load(fp)\n",
    "    \n",
    "with open ('redd_main_time', 'rb') as fp:\n",
    "    main_time_list = pickle.load(fp)\n",
    "    \n",
    "with open('redd_diffreq_time', 'rb') as fp:\n",
    "    diffreq_app_time_list = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now you have \n",
    "- app_time_list\n",
    "- main_time_list\n",
    "- diffreq_app_time_list\n",
    "\n",
    "      diffreq: [[1, 'washer dryer'], [3, 'washer dryer'], [5, 'washer dryer'], [5, 'electric space heater']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Loading data for meter ElecMeterID(instance=1, building=2, dataset='REDD')     "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "nilmtk/utils.py:336: FutureWarning: \n",
      ".resample() is now a deferred operation\n",
      "You called index(...) on this deferred object which materialized it into a series\n",
      "by implicitly taking the mean.  Use .resample(...).mean() instead\n",
      "  return resampled.index[0]\n",
      "nilmtk/utils.py:390: FutureWarning: fill_method is deprecated to .resample()\n",
      "the new syntax is .resample(...).ffill(limit=30)\n",
      "  data = data.resample(**resample_kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data for meter ElecMeterID(instance=2, building=2, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=2, building=4, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=2, building=6, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=2, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=10, building=1, dataset='REDD')     "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "nilmtk/utils.py:390: FutureWarning: fill_method is deprecated to .resample()\n",
      "the new syntax is .resample(...).ffill(limit=17)\n",
      "  data = data.resample(**resample_kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data for meter ElecMeterID(instance=20, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=2, building=3, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=14, building=3, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=2, building=5, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=9, building=5, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n"
     ]
    }
   ],
   "source": [
    "p = 20\n",
    "result_list = [0,0,0,0,0,0]\n",
    "for i in [1, 3, 5]:\n",
    "    main = redd.buildings[i+1].elec.mains().load().next()\n",
    "    length = len(app_time_list[i])\n",
    "    count = 0\n",
    "    result = []\n",
    "    tm = main_time_list[i][0]\n",
    "    for j in range(p, length-1):\n",
    "        time = app_time_list[i][j+1]\n",
    "        #time diff for index j & nan test for main\n",
    "        if time - app_time_list[i][j] <= p and not np.isnan(main.values[time-tm]):\n",
    "            count += 1\n",
    "        else:\n",
    "            result.append([count, j])\n",
    "            count = 0\n",
    "    result.append([count, j])            \n",
    "    result_list[i] = result\n",
    "\n",
    "for i in [0,2,4]:\n",
    "    main = redd.buildings[i+1].elec.mains().load().next()\n",
    "    wash = redd.buildings[i+1].elec['washer dryer'].load().next()\n",
    "    length = len(app_time_list[i])\n",
    "    count = 0\n",
    "    result = []\n",
    "    tm = main_time_list[i][0]\n",
    "    ta = diffreq_app_time_list[i/2][0]\n",
    "    for j in range(p, length-20):\n",
    "        time = app_time_list[i][j+1]\n",
    "        time_wash = (time - ta) / 3\n",
    "        #time diff for index j & nan test for main\n",
    "        if time - app_time_list[i][j] <= p and not np.isnan(main.values[time-tm]) and not np.isnan(wash.values[time_wash]):\n",
    "             count += 1\n",
    "        else:\n",
    "            result.append([count, j])\n",
    "            count = 0\n",
    "    result.append([count, j])            \n",
    "    result_list[i] = result\n",
    "    \n",
    "    \n",
    "ind_list = []\n",
    "for i in range(6):\n",
    "    ind = []\n",
    "    for res in result_list[i]:\n",
    "        if res[0] > 15000:\n",
    "            ind.append(res)\n",
    "    ind_list.append(ind)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ind_list = []\n",
    "for i in range(6):\n",
    "    for res in result_list[i]:\n",
    "        if res[0] > 20000:\n",
    "            ind_list.append([i, res[0], res[1]])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[0, 20917, 25258],\n",
       " [0, 23184, 60439],\n",
       " [0, 25257, 98950],\n",
       " [0, 27078, 126029],\n",
       " [0, 50520, 192424],\n",
       " [0, 24840, 217691],\n",
       " [0, 32540, 288469],\n",
       " [0, 29059, 317529],\n",
       " [0, 22848, 340386],\n",
       " [0, 30326, 475717],\n",
       " [0, 26218, 745857],\n",
       " [1, 31028, 33150],\n",
       " [1, 43382, 82757],\n",
       " [1, 26456, 152497],\n",
       " [1, 21643, 174142],\n",
       " [1, 30298, 204449],\n",
       " [1, 89455, 293915],\n",
       " [1, 20976, 314905],\n",
       " [2, 31921, 88671],\n",
       " [2, 21400, 114360],\n",
       " [2, 26251, 140612],\n",
       " [2, 40738, 201317],\n",
       " [2, 26546, 332921],\n",
       " [2, 70860, 403799],\n",
       " [3, 33028, 113951],\n",
       " [3, 20295, 167285],\n",
       " [3, 38755, 237101],\n",
       " [3, 20471, 260108],\n",
       " [3, 36745, 296854],\n",
       " [3, 97861, 440716],\n",
       " [4, 21669, 80396],\n",
       " [5, 153842, 182940],\n",
       " [5, 40125, 267614],\n",
       " [5, 73400, 341015],\n",
       " [5, 35951, 376966]]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ind_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Missing value filling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_keys =[\n",
    "    ['washer dryer', 'dish washer', 'fridge', 'microwave', 'electric space heater', 'electric stove'],\n",
    "    ['washer dryer', 'dish washer', 'fridge', 'microwave', 'waste disposal unit','electric stove'],\n",
    "    ['washer dryer', 'dish washer', 'fridge', 'microwave', 'electric furnace', 'waste disposal unit', 'CE appliance'],\n",
    "    ['washer dryer', 'dish washer', 'electric furnace', 'electric stove'],\n",
    "    ['washer dryer', 'dish washer', 'fridge', 'microwave', 'electric space heater', 'electric furnace', 'waste disposal unit', 'CE appliance'],\n",
    "    ['washer dryer', 'dish washer', 'fridge', 'electric space heater', 'electric stove', 'CE appliance']\n",
    "]\n",
    "for i in range(6):\n",
    "    data_keys[i].sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building1, iter0\n",
      "Loading data for meter ElecMeterID(instance=2, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=20, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Mon Apr 18 14:03:30 2011', 'Mon Apr 18 14:03:30 2011', 'Mon Apr 18 14:03:30 2011', 'Mon Apr 18 14:03:30 2011', 'Mon Apr 18 14:03:30 2011', 'Mon Apr 18 14:03:30 2011', 'Mon Apr 18 14:03:30 2011']\n",
      "25258\n",
      "building1, iter1\n",
      "Loading data for meter ElecMeterID(instance=2, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=20, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Wed Apr 20 02:29:00 2011', 'Wed Apr 20 02:29:00 2011', 'Wed Apr 20 02:29:00 2011', 'Wed Apr 20 02:29:00 2011', 'Wed Apr 20 02:29:00 2011', 'Wed Apr 20 02:29:00 2011', 'Wed Apr 20 02:29:00 2011']\n",
      "60439\n",
      "building1, iter2\n",
      "Loading data for meter ElecMeterID(instance=2, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=20, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Thu Apr 21 19:41:30 2011', 'Thu Apr 21 19:41:30 2011', 'Thu Apr 21 19:41:30 2011', 'Thu Apr 21 19:41:30 2011', 'Thu Apr 21 19:41:30 2011', 'Thu Apr 21 19:41:30 2011', 'Thu Apr 21 19:41:30 2011']\n",
      "98950\n",
      "building1, iter3\n",
      "Loading data for meter ElecMeterID(instance=2, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=20, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Fri Apr 22 22:48:34 2011', 'Fri Apr 22 22:48:34 2011', 'Fri Apr 22 22:48:34 2011', 'Fri Apr 22 22:48:34 2011', 'Fri Apr 22 22:48:34 2011', 'Fri Apr 22 22:48:33 2011', 'Fri Apr 22 22:48:34 2011']\n",
      "126029\n",
      "building1, iter4\n",
      "Loading data for meter ElecMeterID(instance=2, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=20, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Sun Apr 24 20:47:20 2011', 'Sun Apr 24 20:47:20 2011', 'Sun Apr 24 20:47:20 2011', 'Sun Apr 24 20:47:20 2011', 'Sun Apr 24 20:47:20 2011', 'Sun Apr 24 20:47:18 2011', 'Sun Apr 24 20:47:20 2011']\n",
      "192424\n",
      "building1, iter5\n",
      "Loading data for meter ElecMeterID(instance=2, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=20, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Wed Apr 27 03:22:31 2011', 'Wed Apr 27 03:22:31 2011', 'Wed Apr 27 03:22:31 2011', 'Wed Apr 27 03:22:31 2011', 'Wed Apr 27 03:22:31 2011', 'Wed Apr 27 03:22:30 2011', 'Wed Apr 27 03:22:31 2011']\n",
      "217691\n",
      "building1, iter6\n",
      "Loading data for meter ElecMeterID(instance=2, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=20, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Fri Apr 29 23:10:57 2011', 'Fri Apr 29 23:10:57 2011', 'Fri Apr 29 23:10:57 2011', 'Fri Apr 29 23:10:57 2011', 'Fri Apr 29 23:10:57 2011', 'Fri Apr 29 23:10:57 2011', 'Fri Apr 29 23:10:57 2011']\n",
      "288469\n",
      "building1, iter7\n",
      "Loading data for meter ElecMeterID(instance=2, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=20, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Sun May  1 09:47:25 2011', 'Sun May  1 09:47:25 2011', 'Sun May  1 09:47:25 2011', 'Sun May  1 09:47:25 2011', 'Sun May  1 09:47:25 2011', 'Sun May  1 09:47:24 2011', 'Sun May  1 09:47:25 2011']\n",
      "317529\n",
      "building1, iter8\n",
      "Loading data for meter ElecMeterID(instance=2, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=20, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Mon May  2 17:08:26 2011', 'Mon May  2 17:08:26 2011', 'Mon May  2 17:08:26 2011', 'Mon May  2 17:08:26 2011', 'Mon May  2 17:08:26 2011', 'Mon May  2 17:08:24 2011', 'Mon May  2 17:08:26 2011']\n",
      "340386\n",
      "building1, iter9\n",
      "Loading data for meter ElecMeterID(instance=2, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=20, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Wed May 11 03:20:17 2011', 'Wed May 11 03:20:17 2011', 'Wed May 11 03:20:17 2011', 'Wed May 11 03:20:17 2011', 'Wed May 11 03:20:17 2011', 'Wed May 11 03:20:15 2011', 'Wed May 11 03:20:17 2011']\n",
      "475717\n",
      "building1, iter10\n",
      "Loading data for meter ElecMeterID(instance=2, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=20, building=1, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Mon May 23 10:31:31 2011', 'Mon May 23 10:31:31 2011', 'Mon May 23 10:31:31 2011', 'Mon May 23 10:31:31 2011', 'Mon May 23 10:31:31 2011', 'Mon May 23 10:31:30 2011', 'Mon May 23 10:31:31 2011']\n",
      "745857\n",
      "building2, iter11\n",
      "Loading data for meter ElecMeterID(instance=2, building=2, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Mon Apr 18 03:45:43 2011', 'Mon Apr 18 03:45:43 2011', 'Mon Apr 18 03:45:43 2011', 'Mon Apr 18 03:45:43 2011', 'Mon Apr 18 03:45:43 2011', 'Mon Apr 18 03:45:43 2011', 'Mon Apr 18 03:45:43 2011']\n",
      "33150\n",
      "building2, iter1\n",
      "Loading data for meter ElecMeterID(instance=2, building=2, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Tue Apr 19 20:20:08 2011', 'Tue Apr 19 20:20:08 2011', 'Tue Apr 19 20:20:08 2011', 'Tue Apr 19 20:20:08 2011', 'Tue Apr 19 20:20:08 2011', 'Tue Apr 19 20:20:08 2011', 'Tue Apr 19 20:20:08 2011']\n",
      "82757\n",
      "building2, iter2\n",
      "Loading data for meter ElecMeterID(instance=2, building=2, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Sat Apr 23 17:05:07 2011', 'Sat Apr 23 17:05:07 2011', 'Sat Apr 23 17:05:07 2011', 'Sat Apr 23 17:05:07 2011', 'Sat Apr 23 17:05:07 2011', 'Sat Apr 23 17:05:07 2011', 'Sat Apr 23 17:05:07 2011']\n",
      "152497\n",
      "building2, iter3\n",
      "Loading data for meter ElecMeterID(instance=2, building=2, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Sun Apr 24 20:47:33 2011', 'Sun Apr 24 20:47:33 2011', 'Sun Apr 24 20:47:33 2011', 'Sun Apr 24 20:47:33 2011', 'Sun Apr 24 20:47:33 2011', 'Sun Apr 24 20:47:33 2011', 'Sun Apr 24 20:47:33 2011']\n",
      "174142\n",
      "building2, iter4\n",
      "Loading data for meter ElecMeterID(instance=2, building=2, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Mon Apr 25 19:32:28 2011', 'Mon Apr 25 19:32:28 2011', 'Mon Apr 25 19:32:28 2011', 'Mon Apr 25 19:32:28 2011', 'Mon Apr 25 19:32:28 2011', 'Mon Apr 25 19:32:28 2011', 'Mon Apr 25 19:32:28 2011']\n",
      "204449\n",
      "building2, iter5\n",
      "Loading data for meter ElecMeterID(instance=2, building=2, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Wed Apr 27 03:22:58 2011', 'Wed Apr 27 03:22:58 2011', 'Wed Apr 27 03:22:58 2011', 'Wed Apr 27 03:22:58 2011', 'Wed Apr 27 03:22:58 2011', 'Wed Apr 27 03:22:58 2011', 'Wed Apr 27 03:22:58 2011']\n",
      "293915\n",
      "building2, iter6\n",
      "Loading data for meter ElecMeterID(instance=2, building=2, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Sun May  1 01:07:29 2011', 'Sun May  1 01:07:29 2011', 'Sun May  1 01:07:29 2011', 'Sun May  1 01:07:29 2011', 'Sun May  1 01:07:29 2011', 'Sun May  1 01:07:29 2011', 'Sun May  1 01:07:29 2011']\n",
      "314905\n",
      "building3, iter7\n",
      "Loading data for meter ElecMeterID(instance=2, building=3, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=14, building=3, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Wed Apr 20 08:08:36 2011', 'Wed Apr 20 08:08:36 2011', 'Wed Apr 20 08:08:36 2011', 'Wed Apr 20 08:08:36 2011', 'Wed Apr 20 08:08:36 2011', 'Wed Apr 20 08:08:36 2011', 'Wed Apr 20 08:08:36 2011', 'Wed Apr 20 08:08:36 2011']\n",
      "88671\n",
      "building3, iter1\n",
      "Loading data for meter ElecMeterID(instance=2, building=3, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=14, building=3, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Fri Apr 22 00:13:14 2011', 'Fri Apr 22 00:13:14 2011', 'Fri Apr 22 00:13:14 2011', 'Fri Apr 22 00:13:14 2011', 'Fri Apr 22 00:13:14 2011', 'Fri Apr 22 00:13:12 2011', 'Fri Apr 22 00:13:14 2011', 'Fri Apr 22 00:13:14 2011']\n",
      "114360\n",
      "building3, iter2\n",
      "Loading data for meter ElecMeterID(instance=2, building=3, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=14, building=3, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Fri Apr 22 22:48:38 2011', 'Fri Apr 22 22:48:38 2011', 'Fri Apr 22 22:48:38 2011', 'Fri Apr 22 22:48:38 2011', 'Fri Apr 22 22:48:38 2011', 'Fri Apr 22 22:48:36 2011', 'Fri Apr 22 22:48:38 2011', 'Fri Apr 22 22:48:38 2011']\n",
      "140612\n",
      "building3, iter3\n",
      "Loading data for meter ElecMeterID(instance=2, building=3, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=14, building=3, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Mon Apr 25 08:13:55 2011', 'Mon Apr 25 08:13:55 2011', 'Mon Apr 25 08:13:55 2011', 'Mon Apr 25 08:13:55 2011', 'Mon Apr 25 08:13:55 2011', 'Mon Apr 25 08:13:54 2011', 'Mon Apr 25 08:13:55 2011', 'Mon Apr 25 08:13:55 2011']\n",
      "201317\n",
      "building3, iter4\n",
      "Loading data for meter ElecMeterID(instance=2, building=3, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=14, building=3, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Mon May 23 10:31:30 2011', 'Mon May 23 10:31:30 2011', 'Mon May 23 10:31:30 2011', 'Mon May 23 10:31:30 2011', 'Mon May 23 10:31:30 2011', 'Mon May 23 10:31:30 2011', 'Mon May 23 10:31:30 2011', 'Mon May 23 10:31:30 2011']\n",
      "332921\n",
      "building3, iter5\n",
      "Loading data for meter ElecMeterID(instance=2, building=3, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=14, building=3, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Tue May 24 17:37:17 2011', 'Tue May 24 17:37:17 2011', 'Tue May 24 17:37:17 2011', 'Tue May 24 17:37:17 2011', 'Tue May 24 17:37:17 2011', 'Tue May 24 17:37:15 2011', 'Tue May 24 17:37:17 2011', 'Tue May 24 17:37:17 2011']\n",
      "403799\n",
      "building4, iter6\n",
      "Loading data for meter ElecMeterID(instance=2, building=4, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Fri Apr 22 22:48:36 2011', 'Fri Apr 22 22:48:36 2011', 'Fri Apr 22 22:48:36 2011', 'Fri Apr 22 22:48:36 2011', 'Fri Apr 22 22:48:36 2011']\n",
      "113951\n",
      "building4, iter1\n",
      "Loading data for meter ElecMeterID(instance=2, building=4, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Mon Apr 25 21:04:56 2011', 'Mon Apr 25 21:04:56 2011', 'Mon Apr 25 21:04:56 2011', 'Mon Apr 25 21:04:56 2011', 'Mon Apr 25 21:04:56 2011']\n",
      "167285\n",
      "building4, iter2\n",
      "Loading data for meter ElecMeterID(instance=2, building=4, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Thu Apr 28 03:21:30 2011', 'Thu Apr 28 03:21:30 2011', 'Thu Apr 28 03:21:30 2011', 'Thu Apr 28 03:21:30 2011', 'Thu Apr 28 03:21:30 2011']\n",
      "237101\n",
      "building4, iter3\n",
      "Loading data for meter ElecMeterID(instance=2, building=4, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Fri Apr 29 23:11:53 2011', 'Fri Apr 29 23:11:53 2011', 'Fri Apr 29 23:11:53 2011', 'Fri Apr 29 23:11:53 2011', 'Fri Apr 29 23:11:53 2011']\n",
      "260108\n",
      "building4, iter4\n",
      "Loading data for meter ElecMeterID(instance=2, building=4, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Sat Apr 30 21:01:24 2011', 'Sat Apr 30 21:01:24 2011', 'Sat Apr 30 21:01:24 2011', 'Sat Apr 30 21:01:24 2011', 'Sat Apr 30 21:01:24 2011']\n",
      "296854\n",
      "building4, iter5\n",
      "Loading data for meter ElecMeterID(instance=2, building=4, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Tue May 24 19:49:27 2011', 'Tue May 24 19:49:27 2011', 'Tue May 24 19:49:27 2011', 'Tue May 24 19:49:27 2011', 'Tue May 24 19:49:27 2011']\n",
      "440716\n",
      "building5, iter6\n",
      "Loading data for meter ElecMeterID(instance=2, building=5, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=13, building=5, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "Loading data for meter ElecMeterID(instance=9, building=5, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Mon May 30 21:03:32 2011', 'Mon May 30 21:03:32 2011', 'Mon May 30 21:03:32 2011', 'Mon May 30 21:03:30 2011', 'Mon May 30 21:03:32 2011', 'Mon May 30 21:03:32 2011', 'Mon May 30 21:03:30 2011', 'Mon May 30 21:03:32 2011', 'Mon May 30 21:03:32 2011']\n",
      "80396\n",
      "building6, iter1\n",
      "Loading data for meter ElecMeterID(instance=2, building=6, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Tue May 24 22:25:00 2011', 'Tue May 24 22:25:00 2011', 'Tue May 24 22:25:00 2011', 'Tue May 24 22:25:00 2011', 'Tue May 24 22:25:00 2011', 'Tue May 24 22:25:00 2011', 'Tue May 24 22:25:00 2011']\n",
      "182940\n",
      "building6, iter1\n",
      "Loading data for meter ElecMeterID(instance=2, building=6, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Wed Jun  8 01:53:19 2011', 'Wed Jun  8 01:53:19 2011', 'Wed Jun  8 01:53:19 2011', 'Wed Jun  8 01:53:19 2011', 'Wed Jun  8 01:53:19 2011', 'Wed Jun  8 01:53:19 2011', 'Wed Jun  8 01:53:19 2011']\n",
      "267614\n",
      "building6, iter2\n",
      "Loading data for meter ElecMeterID(instance=2, building=6, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Fri Jun 10 05:07:33 2011', 'Fri Jun 10 05:07:33 2011', 'Fri Jun 10 05:07:33 2011', 'Fri Jun 10 05:07:33 2011', 'Fri Jun 10 05:07:33 2011', 'Fri Jun 10 05:07:33 2011', 'Fri Jun 10 05:07:33 2011']\n",
      "341015\n",
      "building6, iter3\n",
      "Loading data for meter ElecMeterID(instance=2, building=6, dataset='REDD')     \n",
      "Done loading data all meters for this chunk.\n",
      "['Sun Jun 12 16:21:25 2011', 'Sun Jun 12 16:21:25 2011', 'Sun Jun 12 16:21:25 2011', 'Sun Jun 12 16:21:25 2011', 'Sun Jun 12 16:21:25 2011', 'Sun Jun 12 16:21:25 2011', 'Sun Jun 12 16:21:25 2011']\n",
      "376966\n"
     ]
    }
   ],
   "source": [
    "house_iter = 0\n",
    "house_before = 0\n",
    "\n",
    "for index in ind_list:\n",
    "    validation = []\n",
    "    \n",
    "    new_data_list = [[]]\n",
    "    house = index[0]\n",
    "    count = 0\n",
    "    index_end = index[2]\n",
    "    index_start = index_end - index[1] + 1\n",
    "    #l: num_app\n",
    "    l = 1+len(data_keys[house])\n",
    "    print 'building' + str(house+1) + ', iter' + str(house_iter)\n",
    "\n",
    "    #data load in df list\n",
    "    df_list = []\n",
    "    elec = redd.buildings[house+1].elec\n",
    "    main = elec.mains().load().next()\n",
    "    for key in data_keys[house]:\n",
    "        df_list.append(elec[key].load().next())\n",
    "        new_data_list.append([])\n",
    "        \n",
    "        \n",
    "    #j means app no.\n",
    "    for j in range(l-1):\n",
    "        if (house in [0,2,4] and data_keys[house][j] == 'washer dryer') or (house==4 and data_keys[house][j]=='electric space heater'):\n",
    "            app_index = app_time_list[house][index_start] - diffreq_app_time_list[house/2][0]\n",
    "            new_data_list[j].append(df_list[j].values[app_index/3])\n",
    "            \n",
    "            validation.append(df_list[j].index[app_index/3].ctime())\n",
    "        else:\n",
    "            new_data_list[j].append(df_list[j].values[index_start])\n",
    "            \n",
    "            validation.append(df_list[j].index[index_start].ctime())\n",
    "    main_index = app_time_list[house][index_start] - main_time_list[house][0]\n",
    "    new_data_list[l-1].append(main.values[main_index])\n",
    "    \n",
    "    validation.append(main.index[main_index].ctime())\n",
    "    print validation\n",
    "    \n",
    "    # j means app index.\n",
    "    for j in range(index_start+1, index_end+1):\n",
    "        if app_time_list[house][j] - app_time_list[house][j-1] <= 5:\n",
    "            repeat = 1\n",
    "        else:\n",
    "            diff = app_time_list[house][j] - app_time_list[house][j-1]\n",
    "            repeat = diff/3\n",
    "\n",
    "        for o in xrange(repeat):\n",
    "            for k in range(l-1):\n",
    "                if (house in [0,2,4] and data_keys[house][k] == 'washer dryer') or (house==4 and data_keys[house][k]=='electric space heater'):\n",
    "                    app_index = app_time_list[house][j] - diffreq_app_time_list[house/2][0]\n",
    "                    new_data_list[k].append(df_list[k].values[app_index/3])\n",
    "                else:\n",
    "                    new_data_list[k].append(df_list[k].values[j])\n",
    "\n",
    "            main_index = app_time_list[house][j] - main_time_list[house][0]\n",
    "            new_data_list[l-1].append(main.values[main_index])\n",
    "            \n",
    "    print j\n",
    "    \n",
    "    columns = []\n",
    "    for key in data_keys[house]:\n",
    "        columns.append(key)\n",
    "    columns.append('main')\n",
    "    darr = np.array(new_data_list)\n",
    "    n,x,_ = darr.shape\n",
    "    \n",
    "    if house_before != house:\n",
    "        house_before = house\n",
    "        house_iter = 0\n",
    "        \n",
    "    df = pd.DataFrame(columns=columns, data=darr.reshape(n,x).T)\n",
    "    df.to_csv('REDD/redd_house'+str(house+1)+'_'+str(house_iter)+'.csv')\n",
    "    house_iter += 1\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}