src/writing_pandas_hdf5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Writing a Pandas data frame with LAS data to a HDF5 file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Populate data frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: laspy in /usr/local/python/3.10.8/lib/python3.10/site-packages (2.5.1)\n",
      "Requirement already satisfied: numpy in /home/codespace/.local/lib/python3.10/site-packages (from laspy) (1.26.1)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X</th>\n",
       "      <th>Y</th>\n",
       "      <th>Z</th>\n",
       "      <th>intensity</th>\n",
       "      <th>raw_classification</th>\n",
       "      <th>gps_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>555000.0625</td>\n",
       "      <td>4887200.0</td>\n",
       "      <td>120.940003</td>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>467000.43750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>555000.6875</td>\n",
       "      <td>4887199.5</td>\n",
       "      <td>117.330002</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>467000.50000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>555001.3125</td>\n",
       "      <td>4887200.0</td>\n",
       "      <td>115.339996</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>467000.50000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>555000.1875</td>\n",
       "      <td>4887197.0</td>\n",
       "      <td>123.910004</td>\n",
       "      <td>31</td>\n",
       "      <td>1</td>\n",
       "      <td>467000.53125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>555001.9375</td>\n",
       "      <td>4887200.0</td>\n",
       "      <td>111.110001</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>467000.53125</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             X          Y           Z  intensity  raw_classification  \\\n",
       "0  555000.0625  4887200.0  120.940003         30                   1   \n",
       "1  555000.6875  4887199.5  117.330002         22                   1   \n",
       "2  555001.3125  4887200.0  115.339996         10                   1   \n",
       "3  555000.1875  4887197.0  123.910004         31                   1   \n",
       "4  555001.9375  4887200.0  111.110001          8                   1   \n",
       "\n",
       "       gps_time  \n",
       "0  467000.43750  \n",
       "1  467000.50000  \n",
       "2  467000.50000  \n",
       "3  467000.53125  \n",
       "4  467000.53125  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pip install laspy\n",
    "import laspy\n",
    "las = laspy.read(\"../data/test.las\")\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "df = pd.DataFrame({'X': np.array(las.x), 'Y': np.array(las.y), 'Z': np.array(las.z), 'intensity': las.intensity, 'raw_classification': las.raw_classification, 'gps_time': las.gps_time})\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is a built-in HDF export for data frames, see the [Pandas doc](http://pandas.pydata.org/pandas-docs/stable/io.html#hdf5-pytables). However 'pytables' is an implicit dependency.\n",
    "The 'key' identifies the object in the HDF file in case it contains multiple objects. We use 'LAS' as a key to indicate LAS data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tables in /usr/local/python/3.10.8/lib/python3.10/site-packages (3.9.2)\n",
      "Requirement already satisfied: numpy>=1.19.0 in /home/codespace/.local/lib/python3.10/site-packages (from tables) (1.26.1)\n",
      "Requirement already satisfied: numexpr>=2.6.2 in /usr/local/python/3.10.8/lib/python3.10/site-packages (from tables) (2.8.7)\n",
      "Requirement already satisfied: packaging in /home/codespace/.local/lib/python3.10/site-packages (from tables) (23.2)\n",
      "Requirement already satisfied: py-cpuinfo in /usr/local/python/3.10.8/lib/python3.10/site-packages (from tables) (9.0.0)\n",
      "Requirement already satisfied: blosc2>=2.3.0 in /usr/local/python/3.10.8/lib/python3.10/site-packages (from tables) (2.3.1)\n",
      "Requirement already satisfied: ndindex>=1.4 in /usr/local/python/3.10.8/lib/python3.10/site-packages (from blosc2>=2.3.0->tables) (1.7)\n",
      "Requirement already satisfied: msgpack in /usr/local/python/3.10.8/lib/python3.10/site-packages (from blosc2>=2.3.0->tables) (1.0.7)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install tables\n",
    "\n",
    "df.to_hdf('../data/test.h5', mode='w', key='LAS', format='fixed', complib='blosc', complevel=9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The HDF5 file can be compressed (depending on the version of pandas and parameters). But the data contains the coordinates as doubles, whereas LAS stores 4 byte integres (plus offset & scale in the header). We can compare the file sizes for this simple example of the original LAS and the resulting HDF file. Some general perfomrance considerations are given [here](https://pandas.pydata.org/pandas-docs/stable/io.html#performance-considerations)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LAS file: 8.686984062194824 MB\n",
      "HDF file: 6.291470527648926 MB\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "print('LAS file: {} MB'.format(os.stat('../data/test.las').st_size/(1024*1024)))\n",
    "print('HDF file: {} MB'.format(os.stat('../data/test.h5').st_size/(1024*1024)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read the data back"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X</th>\n",
       "      <th>Y</th>\n",
       "      <th>Z</th>\n",
       "      <th>intensity</th>\n",
       "      <th>raw_classification</th>\n",
       "      <th>gps_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>555000.0625</td>\n",
       "      <td>4887200.0</td>\n",
       "      <td>120.940003</td>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>467000.43750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>555000.6875</td>\n",
       "      <td>4887199.5</td>\n",
       "      <td>117.330002</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>467000.50000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>555001.3125</td>\n",
       "      <td>4887200.0</td>\n",
       "      <td>115.339996</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>467000.50000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>555000.1875</td>\n",
       "      <td>4887197.0</td>\n",
       "      <td>123.910004</td>\n",
       "      <td>31</td>\n",
       "      <td>1</td>\n",
       "      <td>467000.53125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>555001.9375</td>\n",
       "      <td>4887200.0</td>\n",
       "      <td>111.110001</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>467000.53125</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             X          Y           Z  intensity  raw_classification  \\\n",
       "0  555000.0625  4887200.0  120.940003         30                   1   \n",
       "1  555000.6875  4887199.5  117.330002         22                   1   \n",
       "2  555001.3125  4887200.0  115.339996         10                   1   \n",
       "3  555000.1875  4887197.0  123.910004         31                   1   \n",
       "4  555001.9375  4887200.0  111.110001          8                   1   \n",
       "\n",
       "       gps_time  \n",
       "0  467000.43750  \n",
       "1  467000.50000  \n",
       "2  467000.50000  \n",
       "3  467000.53125  \n",
       "4  467000.53125  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2 = pd.read_hdf('../data/test.h5', 'LAS')\n",
    "df2.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}