main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "12f347fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "import qlib\n",
    "import pandas as pd\n",
    "from qlib.utils import init_instance_by_config\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b82addca",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[4073920:MainThread](2022-03-01 13:08:42,904) INFO - qlib.Initialization - [config.py:386] - default_conf: client.\n",
      "[4073920:MainThread](2022-03-01 13:08:42,909) WARNING - qlib.Initialization - [config.py:412] - redis connection failed(host=127.0.0.1 port=6379), DiskExpressionCache and DiskDatasetCache will not be used!\n",
      "[4073920:MainThread](2022-03-01 13:08:43,242) INFO - qlib.Initialization - [__init__.py:57] - qlib successfully initialized based on client settings.\n",
      "[4073920:MainThread](2022-03-01 13:08:43,244) INFO - qlib.Initialization - [__init__.py:59] - data_path={'__DEFAULT_FREQ': PosixPath('/storage/qlib/qlib_data/wrds/crsp/a_stock/msf')}\n",
      "[4073920:MainThread](2022-03-01 13:08:44,837) INFO - qlib.timer - [log.py:113] - Time cost: 1.534s | Loading data Done\n",
      "[4073920:MainThread](2022-03-01 13:08:44,840) INFO - qlib.timer - [log.py:113] - Time cost: 0.000s | fit & process data Done\n",
      "[4073920:MainThread](2022-03-01 13:08:44,841) INFO - qlib.timer - [log.py:113] - Time cost: 1.538s | Init data Done\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th colspan=\"18\" halign=\"left\">feature</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>ret</th>\n",
       "      <th>prc</th>\n",
       "      <th>ask</th>\n",
       "      <th>altprc</th>\n",
       "      <th>cfacpr</th>\n",
       "      <th>hsiccd</th>\n",
       "      <th>retx</th>\n",
       "      <th>hexcd</th>\n",
       "      <th>altprcdt</th>\n",
       "      <th>cusip</th>\n",
       "      <th>bidlo</th>\n",
       "      <th>cfacshr</th>\n",
       "      <th>issuno</th>\n",
       "      <th>bid</th>\n",
       "      <th>spread</th>\n",
       "      <th>permco</th>\n",
       "      <th>shrout</th>\n",
       "      <th>vol</th>\n",
       "      <th>askhi</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime</th>\n",
       "      <th>instrument</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1987-06-30</th>\n",
       "      <th>10065.0</th>\n",
       "      <td>0.053254</td>\n",
       "      <td>22.250000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.250000</td>\n",
       "      <td>1.500000</td>\n",
       "      <td>6722.0</td>\n",
       "      <td>0.053254</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1987-06-30</td>\n",
       "      <td>00621210</td>\n",
       "      <td>21.250000</td>\n",
       "      <td>1.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20023.0</td>\n",
       "      <td>24005.0</td>\n",
       "      <td>2055.0</td>\n",
       "      <td>22.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10137.0</th>\n",
       "      <td>0.031041</td>\n",
       "      <td>40.125000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40.125000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4911.0</td>\n",
       "      <td>0.012618</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1987-06-30</td>\n",
       "      <td>01736110</td>\n",
       "      <td>39.375000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20045.0</td>\n",
       "      <td>50868.0</td>\n",
       "      <td>28627.0</td>\n",
       "      <td>41.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10145.0</th>\n",
       "      <td>0.077160</td>\n",
       "      <td>43.625000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>43.625000</td>\n",
       "      <td>4.191136</td>\n",
       "      <td>5099.0</td>\n",
       "      <td>0.077160</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1987-06-30</td>\n",
       "      <td>43851610</td>\n",
       "      <td>40.250000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22168.0</td>\n",
       "      <td>174424.0</td>\n",
       "      <td>105415.0</td>\n",
       "      <td>44.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10225.0</th>\n",
       "      <td>0.071633</td>\n",
       "      <td>46.750000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>46.750000</td>\n",
       "      <td>4.276438</td>\n",
       "      <td>3433.0</td>\n",
       "      <td>0.071633</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1987-06-30</td>\n",
       "      <td>07373010</td>\n",
       "      <td>43.875000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20067.0</td>\n",
       "      <td>110095.0</td>\n",
       "      <td>46749.0</td>\n",
       "      <td>48.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10516.0</th>\n",
       "      <td>0.166667</td>\n",
       "      <td>24.500000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>24.500000</td>\n",
       "      <td>4.677587</td>\n",
       "      <td>2046.0</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1987-06-30</td>\n",
       "      <td>03948310</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>4.677587</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20207.0</td>\n",
       "      <td>164352.0</td>\n",
       "      <td>203211.0</td>\n",
       "      <td>25.625000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2020-12-31</th>\n",
       "      <th>21135.0</th>\n",
       "      <td>0.022362</td>\n",
       "      <td>14.630000</td>\n",
       "      <td>14.650000</td>\n",
       "      <td>14.630000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2851.0</td>\n",
       "      <td>0.022362</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020-12-31</td>\n",
       "      <td>31540510</td>\n",
       "      <td>13.540000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>14.640000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20699.0</td>\n",
       "      <td>82291.0</td>\n",
       "      <td>77320.0</td>\n",
       "      <td>14.970000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21178.0</th>\n",
       "      <td>-0.027452</td>\n",
       "      <td>354.980011</td>\n",
       "      <td>354.950012</td>\n",
       "      <td>354.980011</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3761.0</td>\n",
       "      <td>-0.027452</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020-12-31</td>\n",
       "      <td>53983010</td>\n",
       "      <td>347.920013</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>354.760010</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21110.0</td>\n",
       "      <td>279784.0</td>\n",
       "      <td>351722.0</td>\n",
       "      <td>366.609985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21186.0</th>\n",
       "      <td>0.031272</td>\n",
       "      <td>43.529999</td>\n",
       "      <td>43.529999</td>\n",
       "      <td>43.529999</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2631.0</td>\n",
       "      <td>0.031272</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020-12-31</td>\n",
       "      <td>96145D10</td>\n",
       "      <td>42.860001</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>43.509998</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21913.0</td>\n",
       "      <td>263090.0</td>\n",
       "      <td>352120.0</td>\n",
       "      <td>45.110001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21207.0</th>\n",
       "      <td>0.024991</td>\n",
       "      <td>59.889999</td>\n",
       "      <td>59.900002</td>\n",
       "      <td>59.889999</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1041.0</td>\n",
       "      <td>0.018191</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020-12-31</td>\n",
       "      <td>65163910</td>\n",
       "      <td>57.560001</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>59.889999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21286.0</td>\n",
       "      <td>803358.0</td>\n",
       "      <td>1208521.0</td>\n",
       "      <td>61.639999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21573.0</th>\n",
       "      <td>0.004850</td>\n",
       "      <td>49.720001</td>\n",
       "      <td>49.700001</td>\n",
       "      <td>49.720001</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2621.0</td>\n",
       "      <td>0.004850</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020-12-31</td>\n",
       "      <td>46014610</td>\n",
       "      <td>48.009998</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>49.680000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20994.0</td>\n",
       "      <td>393095.0</td>\n",
       "      <td>527625.0</td>\n",
       "      <td>49.980000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>38175 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          label     feature                                    \\\n",
       "                            ret         prc         ask      altprc    cfacpr   \n",
       "datetime   instrument                                                           \n",
       "1987-06-30 10065.0     0.053254   22.250000         NaN   22.250000  1.500000   \n",
       "           10137.0     0.031041   40.125000         NaN   40.125000  2.000000   \n",
       "           10145.0     0.077160   43.625000         NaN   43.625000  4.191136   \n",
       "           10225.0     0.071633   46.750000         NaN   46.750000  4.276438   \n",
       "           10516.0     0.166667   24.500000         NaN   24.500000  4.677587   \n",
       "...                         ...         ...         ...         ...       ...   \n",
       "2020-12-31 21135.0     0.022362   14.630000   14.650000   14.630000  1.000000   \n",
       "           21178.0    -0.027452  354.980011  354.950012  354.980011  1.000000   \n",
       "           21186.0     0.031272   43.529999   43.529999   43.529999  1.000000   \n",
       "           21207.0     0.024991   59.889999   59.900002   59.889999  1.000000   \n",
       "           21573.0     0.004850   49.720001   49.700001   49.720001  1.000000   \n",
       "\n",
       "                                                                    \\\n",
       "                       hsiccd      retx hexcd   altprcdt     cusip   \n",
       "datetime   instrument                                                \n",
       "1987-06-30 10065.0     6722.0  0.053254   1.0 1987-06-30  00621210   \n",
       "           10137.0     4911.0  0.012618   1.0 1987-06-30  01736110   \n",
       "           10145.0     5099.0  0.077160   1.0 1987-06-30  43851610   \n",
       "           10225.0     3433.0  0.071633   1.0 1987-06-30  07373010   \n",
       "           10516.0     2046.0  0.166667   1.0 1987-06-30  03948310   \n",
       "...                       ...       ...   ...        ...       ...   \n",
       "2020-12-31 21135.0     2851.0  0.022362   1.0 2020-12-31  31540510   \n",
       "           21178.0     3761.0 -0.027452   1.0 2020-12-31  53983010   \n",
       "           21186.0     2631.0  0.031272   1.0 2020-12-31  96145D10   \n",
       "           21207.0     1041.0  0.018191   1.0 2020-12-31  65163910   \n",
       "           21573.0     2621.0  0.004850   1.0 2020-12-31  46014610   \n",
       "\n",
       "                                                                       \\\n",
       "                            bidlo   cfacshr issuno         bid spread   \n",
       "datetime   instrument                                                   \n",
       "1987-06-30 10065.0      21.250000  1.500000    0.0         NaN    NaN   \n",
       "           10137.0      39.375000  2.000000    0.0         NaN    NaN   \n",
       "           10145.0      40.250000  4.000000    0.0         NaN    NaN   \n",
       "           10225.0      43.875000  2.000000    0.0         NaN    NaN   \n",
       "           10516.0      21.000000  4.677587    0.0         NaN    NaN   \n",
       "...                           ...       ...    ...         ...    ...   \n",
       "2020-12-31 21135.0      13.540000  1.000000    0.0   14.640000    NaN   \n",
       "           21178.0     347.920013  1.000000    0.0  354.760010    NaN   \n",
       "           21186.0      42.860001  1.000000    0.0   43.509998    NaN   \n",
       "           21207.0      57.560001  1.000000    0.0   59.889999    NaN   \n",
       "           21573.0      48.009998  1.000000    0.0   49.680000    NaN   \n",
       "\n",
       "                                                                 \n",
       "                        permco    shrout        vol       askhi  \n",
       "datetime   instrument                                            \n",
       "1987-06-30 10065.0     20023.0   24005.0     2055.0   22.500000  \n",
       "           10137.0     20045.0   50868.0    28627.0   41.250000  \n",
       "           10145.0     22168.0  174424.0   105415.0   44.500000  \n",
       "           10225.0     20067.0  110095.0    46749.0   48.250000  \n",
       "           10516.0     20207.0  164352.0   203211.0   25.625000  \n",
       "...                        ...       ...        ...         ...  \n",
       "2020-12-31 21135.0     20699.0   82291.0    77320.0   14.970000  \n",
       "           21178.0     21110.0  279784.0   351722.0  366.609985  \n",
       "           21186.0     21913.0  263090.0   352120.0   45.110001  \n",
       "           21207.0     21286.0  803358.0  1208521.0   61.639999  \n",
       "           21573.0     20994.0  393095.0   527625.0   49.980000  \n",
       "\n",
       "[38175 rows x 19 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CRSP monthly stocks\n",
    "provider_uri=\"/storage/qlib/qlib_data/wrds/crsp/a_stock/msf\"\n",
    "qlib.init(provider_uri=provider_uri)\n",
    "\n",
    "market=\"test100\"\n",
    "data_handler_config = {\n",
    "    \"start_time\": \"1987-06-30\",\n",
    "    \"end_time\": \"2020-12-31\",\n",
    "    \"fit_start_time\": \"2008-01-01\",\n",
    "    \"fit_end_time\": \"2020-12-31\",\n",
    "    \"instruments\": market,\n",
    "    \"label\": ([\"$ret\"], [\"ret\"])\n",
    "}\n",
    "\n",
    "\n",
    "dataset_conifg= {\n",
    "    \"class\": \"DatasetH\",\n",
    "    \"module_path\": \"qlib.data.dataset\",\n",
    "    \"kwargs\": {\n",
    "        \"handler\": {\n",
    "            \"class\": \"WRDS\",\n",
    "            \"module_path\": \"data.wrds_handler\",\n",
    "            \"kwargs\": data_handler_config,\n",
    "        },\n",
    "        \"segments\": {\n",
    "            \"train\": (\"2008-01-01\", \"2014-12-31\"),\n",
    "            \"valid\": (\"2015-01-01\", \"2016-12-31\"),\n",
    "            \"test\": (\"2017-01-01\", \"2020-12-31\"),\n",
    "            \"all\": (\"1987-06-30\", \"2020-12-31\"),\n",
    "        },\n",
    "    },\n",
    "}\n",
    "\n",
    "dataset = init_instance_by_config(dataset_conifg)\n",
    "df=dataset.prepare(\"all\", col_set=[\"feature\", \"label\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "da34cb95",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[4073920:MainThread](2022-03-01 13:08:44,899) INFO - qlib.Initialization - [config.py:386] - default_conf: client.\n",
      "[4073920:MainThread](2022-03-01 13:08:44,902) WARNING - qlib.Initialization - [config.py:412] - redis connection failed(host=127.0.0.1 port=6379), DiskExpressionCache and DiskDatasetCache will not be used!\n",
      "[4073920:MainThread](2022-03-01 13:08:44,903) INFO - qlib.Initialization - [__init__.py:57] - qlib successfully initialized based on client settings.\n",
      "[4073920:MainThread](2022-03-01 13:08:44,904) INFO - qlib.Initialization - [__init__.py:59] - data_path={'__DEFAULT_FREQ': PosixPath('/storage/qlib/qlib_data/wrds/comp/naa/funda')}\n",
      "[4073920:MainThread](2022-03-01 13:08:45,652) INFO - qlib.timer - [log.py:113] - Time cost: 0.718s | Loading data Done\n",
      "[4073920:MainThread](2022-03-01 13:08:45,654) INFO - qlib.timer - [log.py:113] - Time cost: 0.000s | fit & process data Done\n",
      "[4073920:MainThread](2022-03-01 13:08:45,655) INFO - qlib.timer - [log.py:113] - Time cost: 0.721s | Init data Done\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"21\" halign=\"left\">feature</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>txtubpospinc</th>\n",
       "      <th>derllt</th>\n",
       "      <th>islg</th>\n",
       "      <th>aodo</th>\n",
       "      <th>aldo</th>\n",
       "      <th>glced</th>\n",
       "      <th>optprcgr</th>\n",
       "      <th>xnbi</th>\n",
       "      <th>cshrp</th>\n",
       "      <th>ppeveb</th>\n",
       "      <th>...</th>\n",
       "      <th>cfo</th>\n",
       "      <th>dvpsp_f</th>\n",
       "      <th>txtubsettle</th>\n",
       "      <th>txfo</th>\n",
       "      <th>urect</th>\n",
       "      <th>dpc</th>\n",
       "      <th>txtubxintbs</th>\n",
       "      <th>sppiv</th>\n",
       "      <th>bclr</th>\n",
       "      <th>prcad</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime</th>\n",
       "      <th>instrument</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2015-01-31</th>\n",
       "      <th>001045_INDL_STD_C_D</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>001078_INDL_STD_C_D</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>001209_INDL_STD_C_D</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>001300_INDL_STD_C_D</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>001414_INDL_STD_C_D</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">2021-02-28</th>\n",
       "      <th>002435_INDL_STD_C_D</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>011017_INDL_STD_C_D</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">2021-03-31</th>\n",
       "      <th>002435_INDL_STD_C_D</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>011017_INDL_STD_C_D</th>\n",
       "      <td>0.0</td>\n",
       "      <td>25.719</td>\n",
       "      <td>NaN</td>\n",
       "      <td>76.252998</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>996.885986</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0700</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.625999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>44.733002</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-04-30</th>\n",
       "      <th>002435_INDL_STD_C_D</th>\n",
       "      <td>1.0</td>\n",
       "      <td>17.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>251.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>69.209999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1691.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.7076</td>\n",
       "      <td>1.0</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>77.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-127.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7148 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    feature                                \\\n",
       "                               txtubpospinc  derllt islg        aodo aldo   \n",
       "datetime   instrument                                                       \n",
       "2015-01-31 001045_INDL_STD_C_D          NaN     NaN  NaN         NaN  NaN   \n",
       "           001078_INDL_STD_C_D          NaN     NaN  NaN         NaN  NaN   \n",
       "           001209_INDL_STD_C_D          NaN     NaN  NaN         NaN  NaN   \n",
       "           001300_INDL_STD_C_D          NaN     NaN  NaN         NaN  NaN   \n",
       "           001414_INDL_STD_C_D          NaN     NaN  NaN         NaN  NaN   \n",
       "...                                     ...     ...  ...         ...  ...   \n",
       "2021-02-28 002435_INDL_STD_C_D          NaN     NaN  NaN         NaN  NaN   \n",
       "           011017_INDL_STD_C_D          NaN     NaN  NaN         NaN  NaN   \n",
       "2021-03-31 002435_INDL_STD_C_D          NaN     NaN  NaN         NaN  NaN   \n",
       "           011017_INDL_STD_C_D          0.0  25.719  NaN   76.252998  0.0   \n",
       "2021-04-30 002435_INDL_STD_C_D          1.0  17.000  NaN  251.000000  0.0   \n",
       "\n",
       "                                                                         ...  \\\n",
       "                               glced   optprcgr xnbi cshrp       ppeveb  ...   \n",
       "datetime   instrument                                                    ...   \n",
       "2015-01-31 001045_INDL_STD_C_D   NaN        NaN  NaN   NaN          NaN  ...   \n",
       "           001078_INDL_STD_C_D   NaN        NaN  NaN   NaN          NaN  ...   \n",
       "           001209_INDL_STD_C_D   NaN        NaN  NaN   NaN          NaN  ...   \n",
       "           001300_INDL_STD_C_D   NaN        NaN  NaN   NaN          NaN  ...   \n",
       "           001414_INDL_STD_C_D   NaN        NaN  NaN   NaN          NaN  ...   \n",
       "...                              ...        ...  ...   ...          ...  ...   \n",
       "2021-02-28 002435_INDL_STD_C_D   NaN        NaN  NaN   NaN          NaN  ...   \n",
       "           011017_INDL_STD_C_D   NaN        NaN  NaN   NaN          NaN  ...   \n",
       "2021-03-31 002435_INDL_STD_C_D   NaN        NaN  NaN   NaN          NaN  ...   \n",
       "           011017_INDL_STD_C_D   NaN        NaN  NaN   NaN   996.885986  ...   \n",
       "2021-04-30 002435_INDL_STD_C_D   NaN  69.209999  NaN   NaN  1691.000000  ...   \n",
       "\n",
       "                                                                         \\\n",
       "                               cfo dvpsp_f txtubsettle       txfo urect   \n",
       "datetime   instrument                                                     \n",
       "2015-01-31 001045_INDL_STD_C_D NaN     NaN         NaN        NaN   NaN   \n",
       "           001078_INDL_STD_C_D NaN     NaN         NaN        NaN   NaN   \n",
       "           001209_INDL_STD_C_D NaN     NaN         NaN        NaN   NaN   \n",
       "           001300_INDL_STD_C_D NaN     NaN         NaN        NaN   NaN   \n",
       "           001414_INDL_STD_C_D NaN     NaN         NaN        NaN   NaN   \n",
       "...                             ..     ...         ...        ...   ...   \n",
       "2021-02-28 002435_INDL_STD_C_D NaN     NaN         NaN        NaN   NaN   \n",
       "           011017_INDL_STD_C_D NaN     NaN         NaN        NaN   NaN   \n",
       "2021-03-31 002435_INDL_STD_C_D NaN     NaN         NaN        NaN   NaN   \n",
       "           011017_INDL_STD_C_D NaN  3.0700         0.0  21.625999   NaN   \n",
       "2021-04-30 002435_INDL_STD_C_D NaN  0.7076         1.0  50.000000   NaN   \n",
       "\n",
       "                                                                         \n",
       "                                      dpc txtubxintbs  sppiv bclr prcad  \n",
       "datetime   instrument                                                    \n",
       "2015-01-31 001045_INDL_STD_C_D        NaN         NaN    NaN  NaN   NaN  \n",
       "           001078_INDL_STD_C_D        NaN         NaN    NaN  NaN   NaN  \n",
       "           001209_INDL_STD_C_D        NaN         NaN    NaN  NaN   NaN  \n",
       "           001300_INDL_STD_C_D        NaN         NaN    NaN  NaN   NaN  \n",
       "           001414_INDL_STD_C_D        NaN         NaN    NaN  NaN   NaN  \n",
       "...                                   ...         ...    ...  ...   ...  \n",
       "2021-02-28 002435_INDL_STD_C_D        NaN         NaN    NaN  NaN   NaN  \n",
       "           011017_INDL_STD_C_D        NaN         NaN    NaN  NaN   NaN  \n",
       "2021-03-31 002435_INDL_STD_C_D        NaN         NaN    NaN  NaN   NaN  \n",
       "           011017_INDL_STD_C_D  44.733002         NaN    0.0  NaN   NaN  \n",
       "2021-04-30 002435_INDL_STD_C_D  77.000000         NaN -127.0  NaN   NaN  \n",
       "\n",
       "[7148 rows x 22 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compustat fundamentals\n",
    "provider_uri=\"/storage/qlib/qlib_data/wrds/comp/naa/funda\"\n",
    "qlib.init(provider_uri=provider_uri)\n",
    "_DEFAUFLT_FUNDA_FIELDS = ['txtubpospinc', 'derllt', 'islg','aodo', 'aldo', 'glced', 'optprcgr', 'xnbi', 'cshrp', 'ppeveb', 'che',\n",
    " 'cld3', 'cfo', 'dvpsp_f', 'txtubsettle', 'txfo', 'urect', 'dpc', 'txtubxintbs', 'sppiv', 'bclr', 'prcad']\n",
    "market=\"test100\"\n",
    "data_handler_config = {\n",
    "    \"start_time\": \"2015-01-01\",\n",
    "    \"end_time\": \"2021-05-31\",\n",
    "    \"fit_start_time\": \"2015-01-01\",\n",
    "    \"fit_end_time\": \"2021-05-31\",\n",
    "    \"instruments\": market,\n",
    "    \"include_fields\": _DEFAUFLT_FUNDA_FIELDS,\n",
    "}\n",
    "\n",
    "\n",
    "dataset_conifg= {\n",
    "    \"class\": \"DatasetH\",\n",
    "    \"module_path\": \"qlib.data.dataset\",\n",
    "    \"kwargs\": {\n",
    "        \"handler\": {\n",
    "            \"class\": \"WRDS\",\n",
    "            \"module_path\": \"data.wrds_handler\",\n",
    "            \"kwargs\": data_handler_config,\n",
    "        },\n",
    "        \"segments\": {\n",
    "            \"train\": (\"2015-01-01\", \"2017-12-31\"),\n",
    "            \"valid\": (\"2017-01-01\", \"2019-12-31\"),\n",
    "            \"test\": (\"2019-01-01\", \"2021-05-31\"),\n",
    "            \"all\": (\"2015-01-01\", \"2021-05-31\"),\n",
    "        },\n",
    "    },\n",
    "}\n",
    "\n",
    "dataset = init_instance_by_config(dataset_conifg)\n",
    "df=dataset.prepare(\"all\", col_set=[\"feature\", \"label\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a50f88ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[4073920:MainThread](2022-03-01 13:08:45,716) INFO - qlib.Initialization - [config.py:386] - default_conf: client.\n",
      "[4073920:MainThread](2022-03-01 13:08:45,718) WARNING - qlib.Initialization - [config.py:412] - redis connection failed(host=127.0.0.1 port=6379), DiskExpressionCache and DiskDatasetCache will not be used!\n",
      "[4073920:MainThread](2022-03-01 13:08:45,720) INFO - qlib.Initialization - [__init__.py:57] - qlib successfully initialized based on client settings.\n",
      "[4073920:MainThread](2022-03-01 13:08:45,720) INFO - qlib.Initialization - [__init__.py:59] - data_path={'__DEFAULT_FREQ': PosixPath('/storage/qlib/qlib_data/wrds/comp/naa/secm')}\n",
      "[4073920:MainThread](2022-03-01 13:08:47,103) INFO - qlib.timer - [log.py:113] - Time cost: 1.271s | Loading data Done\n",
      "[4073920:MainThread](2022-03-01 13:08:47,106) INFO - qlib.timer - [log.py:113] - Time cost: 0.000s | fit & process data Done\n",
      "[4073920:MainThread](2022-03-01 13:08:47,107) INFO - qlib.timer - [log.py:113] - Time cost: 1.274s | Init data Done\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"21\" halign=\"left\">feature</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>isalrt</th>\n",
       "      <th>prchm</th>\n",
       "      <th>sphtic</th>\n",
       "      <th>cshoq</th>\n",
       "      <th>sphmid</th>\n",
       "      <th>secstat</th>\n",
       "      <th>curcdm</th>\n",
       "      <th>dvrate</th>\n",
       "      <th>primiss</th>\n",
       "      <th>gvkey</th>\n",
       "      <th>...</th>\n",
       "      <th>sphiid</th>\n",
       "      <th>mkvalincl</th>\n",
       "      <th>dvpsxm</th>\n",
       "      <th>navm</th>\n",
       "      <th>sphsec</th>\n",
       "      <th>trfm</th>\n",
       "      <th>cshom</th>\n",
       "      <th>prccm</th>\n",
       "      <th>sph100</th>\n",
       "      <th>tpci</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime</th>\n",
       "      <th>instrument</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1987-06-30</th>\n",
       "      <th>001209_01</th>\n",
       "      <td>None</td>\n",
       "      <td>50.250000</td>\n",
       "      <td>None</td>\n",
       "      <td>55.994999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A</td>\n",
       "      <td>USD</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>P</td>\n",
       "      <td>001209</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0.25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.328920</td>\n",
       "      <td>NaN</td>\n",
       "      <td>47.000000</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>001254_01</th>\n",
       "      <td>None</td>\n",
       "      <td>59.250000</td>\n",
       "      <td>None</td>\n",
       "      <td>25.153000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A</td>\n",
       "      <td>USD</td>\n",
       "      <td>1.360000</td>\n",
       "      <td>P</td>\n",
       "      <td>001254</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.070290</td>\n",
       "      <td>NaN</td>\n",
       "      <td>55.250000</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>001440_01</th>\n",
       "      <td>None</td>\n",
       "      <td>28.500000</td>\n",
       "      <td>None</td>\n",
       "      <td>193.535004</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A</td>\n",
       "      <td>USD</td>\n",
       "      <td>2.259999</td>\n",
       "      <td>P</td>\n",
       "      <td>001440</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.262101</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27.500000</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>001526_01</th>\n",
       "      <td>None</td>\n",
       "      <td>36.500000</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A</td>\n",
       "      <td>USD</td>\n",
       "      <td>1.320000</td>\n",
       "      <td>P</td>\n",
       "      <td>001526</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0.33</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.044421</td>\n",
       "      <td>NaN</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>001722_01</th>\n",
       "      <td>None</td>\n",
       "      <td>25.999949</td>\n",
       "      <td>None</td>\n",
       "      <td>164.442001</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A</td>\n",
       "      <td>USD</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>P</td>\n",
       "      <td>001722</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.558723</td>\n",
       "      <td>NaN</td>\n",
       "      <td>24.499952</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2021-06-30</th>\n",
       "      <th>011465_01</th>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A</td>\n",
       "      <td>None</td>\n",
       "      <td>5.600000</td>\n",
       "      <td>P</td>\n",
       "      <td>011465</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>011550_01</th>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A</td>\n",
       "      <td>None</td>\n",
       "      <td>2.710000</td>\n",
       "      <td>P</td>\n",
       "      <td>011550</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>011566_01</th>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A</td>\n",
       "      <td>None</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>P</td>\n",
       "      <td>011566</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0.10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>011636_01</th>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>P</td>\n",
       "      <td>011636</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0.25</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>028590_01</th>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A</td>\n",
       "      <td>None</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>P</td>\n",
       "      <td>028590</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>40900 rows × 44 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      feature                                               \\\n",
       "                       isalrt      prchm sphtic       cshoq sphmid secstat   \n",
       "datetime   instrument                                                        \n",
       "1987-06-30 001209_01     None  50.250000   None   55.994999    NaN       A   \n",
       "           001254_01     None  59.250000   None   25.153000    NaN       A   \n",
       "           001440_01     None  28.500000   None  193.535004    NaN       A   \n",
       "           001526_01     None  36.500000   None         NaN    NaN       A   \n",
       "           001722_01     None  25.999949   None  164.442001    NaN       A   \n",
       "...                       ...        ...    ...         ...    ...     ...   \n",
       "2021-06-30 011465_01     None        NaN   None         NaN    NaN       A   \n",
       "           011550_01     None        NaN   None         NaN    NaN       A   \n",
       "           011566_01     None        NaN   None         NaN    NaN       A   \n",
       "           011636_01     None        NaN   None         NaN    NaN       A   \n",
       "           028590_01     None        NaN   None         NaN    NaN       A   \n",
       "\n",
       "                                                        ...                   \\\n",
       "                      curcdm    dvrate primiss   gvkey  ... sphiid mkvalincl   \n",
       "datetime   instrument                                   ...                    \n",
       "1987-06-30 001209_01     USD  1.000000       P  001209  ...    NaN      None   \n",
       "           001254_01     USD  1.360000       P  001254  ...    NaN      None   \n",
       "           001440_01     USD  2.259999       P  001440  ...    NaN      None   \n",
       "           001526_01     USD  1.320000       P  001526  ...    NaN      None   \n",
       "           001722_01     USD  0.100000       P  001722  ...    NaN      None   \n",
       "...                      ...       ...     ...     ...  ...    ...       ...   \n",
       "2021-06-30 011465_01    None  5.600000       P  011465  ...    NaN      None   \n",
       "           011550_01    None  2.710000       P  011550  ...    NaN      None   \n",
       "           011566_01    None  0.400000       P  011566  ...    NaN      None   \n",
       "           011636_01    None  1.000000       P  011636  ...    NaN      None   \n",
       "           028590_01    None  0.800000       P  028590  ...    NaN      None   \n",
       "\n",
       "                                                                            \\\n",
       "                      dvpsxm navm sphsec      trfm cshom      prccm sph100   \n",
       "datetime   instrument                                                        \n",
       "1987-06-30 001209_01    0.25  NaN    NaN  1.328920   NaN  47.000000   None   \n",
       "           001254_01     NaN  NaN    NaN  3.070290   NaN  55.250000   None   \n",
       "           001440_01     NaN  NaN    NaN  7.262101   NaN  27.500000   None   \n",
       "           001526_01    0.33  NaN    NaN  2.044421   NaN  35.000000   None   \n",
       "           001722_01     NaN  NaN    NaN  1.558723   NaN  24.499952   None   \n",
       "...                      ...  ...    ...       ...   ...        ...    ...   \n",
       "2021-06-30 011465_01     NaN  NaN    NaN       NaN   NaN        NaN   None   \n",
       "           011550_01     NaN  NaN    NaN       NaN   NaN        NaN   None   \n",
       "           011566_01    0.10  NaN    NaN       NaN   NaN        NaN   None   \n",
       "           011636_01    0.25  NaN    NaN       NaN   NaN        NaN   None   \n",
       "           028590_01     NaN  NaN    NaN       NaN   NaN        NaN   None   \n",
       "\n",
       "                            \n",
       "                      tpci  \n",
       "datetime   instrument       \n",
       "1987-06-30 001209_01     0  \n",
       "           001254_01     0  \n",
       "           001440_01     0  \n",
       "           001526_01     0  \n",
       "           001722_01     0  \n",
       "...                    ...  \n",
       "2021-06-30 011465_01     0  \n",
       "           011550_01     0  \n",
       "           011566_01     0  \n",
       "           011636_01     0  \n",
       "           028590_01     0  \n",
       "\n",
       "[40900 rows x 44 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compustat price data\n",
    "provider_uri=\"/storage/qlib/qlib_data/wrds/comp/naa/secm\"\n",
    "qlib.init(provider_uri=provider_uri)\n",
    "\n",
    "market=\"test100\"\n",
    "data_handler_config = {\n",
    "    \"start_time\": \"1987-06-30\",\n",
    "    \"end_time\": \"2021-06-30\",\n",
    "    \"fit_start_time\": \"2008-01-01\",\n",
    "    \"fit_end_time\": \"2021-06-30\",\n",
    "    \"instruments\": market,\n",
    "}\n",
    "\n",
    "\n",
    "dataset_conifg= {\n",
    "    \"class\": \"DatasetH\",\n",
    "    \"module_path\": \"qlib.data.dataset\",\n",
    "    \"kwargs\": {\n",
    "        \"handler\": {\n",
    "            \"class\": \"WRDS\",\n",
    "            \"module_path\": \"data.wrds_handler\",\n",
    "            \"kwargs\": data_handler_config,\n",
    "        },\n",
    "        \"segments\": {\n",
    "            \"train\": (\"2008-01-01\", \"2014-12-31\"),\n",
    "            \"valid\": (\"2015-01-01\", \"2016-12-31\"),\n",
    "            \"test\": (\"2017-01-01\", \"2021-06-30\"),\n",
    "            \"all\": (\"1987-06-30\", \"2021-06-30\"),\n",
    "        },\n",
    "    },\n",
    "}\n",
    "\n",
    "dataset = init_instance_by_config(dataset_conifg)\n",
    "df=dataset.prepare(\"all\", col_set=[\"feature\", \"label\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ee908f8f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[4073920:MainThread](2022-03-01 13:08:47,214) INFO - qlib.Initialization - [config.py:386] - default_conf: client.\n",
      "[4073920:MainThread](2022-03-01 13:08:47,217) WARNING - qlib.Initialization - [config.py:412] - redis connection failed(host=127.0.0.1 port=6379), DiskExpressionCache and DiskDatasetCache will not be used!\n",
      "[4073920:MainThread](2022-03-01 13:08:47,219) INFO - qlib.Initialization - [__init__.py:57] - qlib successfully initialized based on client settings.\n",
      "[4073920:MainThread](2022-03-01 13:08:47,221) INFO - qlib.Initialization - [__init__.py:59] - data_path={'__DEFAULT_FREQ': PosixPath('/storage/qlib/qlib_data/wrds/comp/d_global/g_funda')}\n",
      "[4073920:MainThread](2022-03-01 13:10:13,818) INFO - qlib.timer - [log.py:113] - Time cost: 54.587s | Loading data Done\n",
      "[4073920:MainThread](2022-03-01 13:10:13,820) INFO - qlib.timer - [log.py:113] - Time cost: 0.000s | fit & process data Done\n",
      "[4073920:MainThread](2022-03-01 13:10:13,821) INFO - qlib.timer - [log.py:113] - Time cost: 54.590s | Init data Done\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"21\" halign=\"left\">feature</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>curcd</th>\n",
       "      <th>loc</th>\n",
       "      <th>acqmeth</th>\n",
       "      <th>compst</th>\n",
       "      <th>bspr</th>\n",
       "      <th>popsrc</th>\n",
       "      <th>auop</th>\n",
       "      <th>final</th>\n",
       "      <th>indfmt</th>\n",
       "      <th>consol</th>\n",
       "      <th>...</th>\n",
       "      <th>curcdi</th>\n",
       "      <th>acctstd</th>\n",
       "      <th>iid</th>\n",
       "      <th>fic</th>\n",
       "      <th>isin</th>\n",
       "      <th>stalt</th>\n",
       "      <th>datafmt</th>\n",
       "      <th>costat</th>\n",
       "      <th>au</th>\n",
       "      <th>conm</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime</th>\n",
       "      <th>instrument</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1987-06-30</th>\n",
       "      <th>015745_FS_HIST_STD_C_I</th>\n",
       "      <td>ITL</td>\n",
       "      <td>ITA</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>GB</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>Y</td>\n",
       "      <td>FS</td>\n",
       "      <td>C</td>\n",
       "      <td>...</td>\n",
       "      <td>ITL</td>\n",
       "      <td>DS</td>\n",
       "      <td>03W</td>\n",
       "      <td>ITA</td>\n",
       "      <td>IT0003121677</td>\n",
       "      <td>None</td>\n",
       "      <td>HIST_STD</td>\n",
       "      <td>A</td>\n",
       "      <td>1</td>\n",
       "      <td>CREDITO EMILIANO SPA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>015756_FS_HIST_STD_C_I</th>\n",
       "      <td>AUD</td>\n",
       "      <td>AUS</td>\n",
       "      <td>None</td>\n",
       "      <td>AA</td>\n",
       "      <td>GO</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>Y</td>\n",
       "      <td>FS</td>\n",
       "      <td>C</td>\n",
       "      <td>...</td>\n",
       "      <td>AUD</td>\n",
       "      <td>DS</td>\n",
       "      <td>01W</td>\n",
       "      <td>AUS</td>\n",
       "      <td>AU000000QBE9</td>\n",
       "      <td>None</td>\n",
       "      <td>HIST_STD</td>\n",
       "      <td>A</td>\n",
       "      <td>3</td>\n",
       "      <td>QBE INSURANCE GROUP LTD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100988_INDL_HIST_STD_C_I</th>\n",
       "      <td>JPY</td>\n",
       "      <td>JPN</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>GO</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>Y</td>\n",
       "      <td>INDL</td>\n",
       "      <td>C</td>\n",
       "      <td>...</td>\n",
       "      <td>JPY</td>\n",
       "      <td>DS</td>\n",
       "      <td>01W</td>\n",
       "      <td>JPN</td>\n",
       "      <td>JP3491000000</td>\n",
       "      <td>None</td>\n",
       "      <td>HIST_STD</td>\n",
       "      <td>A</td>\n",
       "      <td>9</td>\n",
       "      <td>DAIDO STEEL CO LTD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201879_INDL_HIST_STD_C_I</th>\n",
       "      <td>JPY</td>\n",
       "      <td>JPN</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>GO</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>Y</td>\n",
       "      <td>INDL</td>\n",
       "      <td>C</td>\n",
       "      <td>...</td>\n",
       "      <td>JPY</td>\n",
       "      <td>DS</td>\n",
       "      <td>01W</td>\n",
       "      <td>JPN</td>\n",
       "      <td>JP3142200009</td>\n",
       "      <td>None</td>\n",
       "      <td>HIST_STD</td>\n",
       "      <td>A</td>\n",
       "      <td>9</td>\n",
       "      <td>ICHIMASA KAMABOKO CO LTD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221819_INDL_HIST_STD_C_I</th>\n",
       "      <td>JPY</td>\n",
       "      <td>JPN</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>GO</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>Y</td>\n",
       "      <td>INDL</td>\n",
       "      <td>C</td>\n",
       "      <td>...</td>\n",
       "      <td>JPY</td>\n",
       "      <td>DS</td>\n",
       "      <td>01W</td>\n",
       "      <td>JPN</td>\n",
       "      <td>JP3695000004</td>\n",
       "      <td>None</td>\n",
       "      <td>HIST_STD</td>\n",
       "      <td>A</td>\n",
       "      <td>9</td>\n",
       "      <td>NIPPON KANZAI CO LTD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2020-12-31</th>\n",
       "      <th>205456_INDL_HIST_STD_C_I</th>\n",
       "      <td>JPY</td>\n",
       "      <td>JPN</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>GO</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>INDL</td>\n",
       "      <td>C</td>\n",
       "      <td>...</td>\n",
       "      <td>JPY</td>\n",
       "      <td>DS</td>\n",
       "      <td>01W</td>\n",
       "      <td>JPN</td>\n",
       "      <td>JP3172800009</td>\n",
       "      <td>None</td>\n",
       "      <td>HIST_STD</td>\n",
       "      <td>A</td>\n",
       "      <td>9</td>\n",
       "      <td>ONAMBA CO LTD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208310_INDL_HIST_STD_C_I</th>\n",
       "      <td>JPY</td>\n",
       "      <td>JPN</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>GO</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>INDL</td>\n",
       "      <td>C</td>\n",
       "      <td>...</td>\n",
       "      <td>JPY</td>\n",
       "      <td>DS</td>\n",
       "      <td>01W</td>\n",
       "      <td>JPN</td>\n",
       "      <td>JP3424600009</td>\n",
       "      <td>None</td>\n",
       "      <td>HIST_STD</td>\n",
       "      <td>A</td>\n",
       "      <td>9</td>\n",
       "      <td>SENSHUKAI CO LTD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209063_INDL_HIST_STD_C_I</th>\n",
       "      <td>JPY</td>\n",
       "      <td>JPN</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>GO</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>INDL</td>\n",
       "      <td>C</td>\n",
       "      <td>...</td>\n",
       "      <td>JPY</td>\n",
       "      <td>DS</td>\n",
       "      <td>01W</td>\n",
       "      <td>JPN</td>\n",
       "      <td>JP3552170007</td>\n",
       "      <td>None</td>\n",
       "      <td>HIST_STD</td>\n",
       "      <td>A</td>\n",
       "      <td>9</td>\n",
       "      <td>TOABO CORPORATION</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209104_INDL_HIST_STD_C_I</th>\n",
       "      <td>JPY</td>\n",
       "      <td>JPN</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>GO</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>INDL</td>\n",
       "      <td>C</td>\n",
       "      <td>...</td>\n",
       "      <td>JPY</td>\n",
       "      <td>DS</td>\n",
       "      <td>01W</td>\n",
       "      <td>JPN</td>\n",
       "      <td>JP3571800006</td>\n",
       "      <td>None</td>\n",
       "      <td>HIST_STD</td>\n",
       "      <td>A</td>\n",
       "      <td>9</td>\n",
       "      <td>TOKYO OHKA KOGYO CO LTD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210693_INDL_HIST_STD_C_I</th>\n",
       "      <td>JPY</td>\n",
       "      <td>JPN</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>GO</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>INDL</td>\n",
       "      <td>C</td>\n",
       "      <td>...</td>\n",
       "      <td>JPY</td>\n",
       "      <td>DS</td>\n",
       "      <td>01W</td>\n",
       "      <td>JPN</td>\n",
       "      <td>JP3801600002</td>\n",
       "      <td>None</td>\n",
       "      <td>HIST_STD</td>\n",
       "      <td>A</td>\n",
       "      <td>9</td>\n",
       "      <td>PIGEON CORP</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>39266 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    feature                                  \\\n",
       "                                      curcd  loc acqmeth compst bspr popsrc   \n",
       "datetime   instrument                                                         \n",
       "1987-06-30 015745_FS_HIST_STD_C_I       ITL  ITA    None   None   GB      I   \n",
       "           015756_FS_HIST_STD_C_I       AUD  AUS    None     AA   GO      I   \n",
       "           100988_INDL_HIST_STD_C_I     JPY  JPN    None   None   GO      I   \n",
       "           201879_INDL_HIST_STD_C_I     JPY  JPN    None   None   GO      I   \n",
       "           221819_INDL_HIST_STD_C_I     JPY  JPN    None   None   GO      I   \n",
       "...                                     ...  ...     ...    ...  ...    ...   \n",
       "2020-12-31 205456_INDL_HIST_STD_C_I     JPY  JPN    None   None   GO      I   \n",
       "           208310_INDL_HIST_STD_C_I     JPY  JPN    None   None   GO      I   \n",
       "           209063_INDL_HIST_STD_C_I     JPY  JPN    None   None   GO      I   \n",
       "           209104_INDL_HIST_STD_C_I     JPY  JPN    None   None   GO      I   \n",
       "           210693_INDL_HIST_STD_C_I     JPY  JPN    None   None   GO      I   \n",
       "\n",
       "                                                              ...         \\\n",
       "                                    auop final indfmt consol  ... curcdi   \n",
       "datetime   instrument                                         ...          \n",
       "1987-06-30 015745_FS_HIST_STD_C_I      1     Y     FS      C  ...    ITL   \n",
       "           015756_FS_HIST_STD_C_I      1     Y     FS      C  ...    AUD   \n",
       "           100988_INDL_HIST_STD_C_I    1     Y   INDL      C  ...    JPY   \n",
       "           201879_INDL_HIST_STD_C_I    1     Y   INDL      C  ...    JPY   \n",
       "           221819_INDL_HIST_STD_C_I    1     Y   INDL      C  ...    JPY   \n",
       "...                                  ...   ...    ...    ...  ...    ...   \n",
       "2020-12-31 205456_INDL_HIST_STD_C_I    1  None   INDL      C  ...    JPY   \n",
       "           208310_INDL_HIST_STD_C_I    1  None   INDL      C  ...    JPY   \n",
       "           209063_INDL_HIST_STD_C_I    1  None   INDL      C  ...    JPY   \n",
       "           209104_INDL_HIST_STD_C_I    1  None   INDL      C  ...    JPY   \n",
       "           210693_INDL_HIST_STD_C_I    1  None   INDL      C  ...    JPY   \n",
       "\n",
       "                                                                           \\\n",
       "                                    acctstd  iid  fic          isin stalt   \n",
       "datetime   instrument                                                       \n",
       "1987-06-30 015745_FS_HIST_STD_C_I        DS  03W  ITA  IT0003121677  None   \n",
       "           015756_FS_HIST_STD_C_I        DS  01W  AUS  AU000000QBE9  None   \n",
       "           100988_INDL_HIST_STD_C_I      DS  01W  JPN  JP3491000000  None   \n",
       "           201879_INDL_HIST_STD_C_I      DS  01W  JPN  JP3142200009  None   \n",
       "           221819_INDL_HIST_STD_C_I      DS  01W  JPN  JP3695000004  None   \n",
       "...                                     ...  ...  ...           ...   ...   \n",
       "2020-12-31 205456_INDL_HIST_STD_C_I      DS  01W  JPN  JP3172800009  None   \n",
       "           208310_INDL_HIST_STD_C_I      DS  01W  JPN  JP3424600009  None   \n",
       "           209063_INDL_HIST_STD_C_I      DS  01W  JPN  JP3552170007  None   \n",
       "           209104_INDL_HIST_STD_C_I      DS  01W  JPN  JP3571800006  None   \n",
       "           210693_INDL_HIST_STD_C_I      DS  01W  JPN  JP3801600002  None   \n",
       "\n",
       "                                                         \\\n",
       "                                      datafmt costat au   \n",
       "datetime   instrument                                     \n",
       "1987-06-30 015745_FS_HIST_STD_C_I    HIST_STD      A  1   \n",
       "           015756_FS_HIST_STD_C_I    HIST_STD      A  3   \n",
       "           100988_INDL_HIST_STD_C_I  HIST_STD      A  9   \n",
       "           201879_INDL_HIST_STD_C_I  HIST_STD      A  9   \n",
       "           221819_INDL_HIST_STD_C_I  HIST_STD      A  9   \n",
       "...                                       ...    ... ..   \n",
       "2020-12-31 205456_INDL_HIST_STD_C_I  HIST_STD      A  9   \n",
       "           208310_INDL_HIST_STD_C_I  HIST_STD      A  9   \n",
       "           209063_INDL_HIST_STD_C_I  HIST_STD      A  9   \n",
       "           209104_INDL_HIST_STD_C_I  HIST_STD      A  9   \n",
       "           210693_INDL_HIST_STD_C_I  HIST_STD      A  9   \n",
       "\n",
       "                                                               \n",
       "                                                         conm  \n",
       "datetime   instrument                                          \n",
       "1987-06-30 015745_FS_HIST_STD_C_I        CREDITO EMILIANO SPA  \n",
       "           015756_FS_HIST_STD_C_I     QBE INSURANCE GROUP LTD  \n",
       "           100988_INDL_HIST_STD_C_I        DAIDO STEEL CO LTD  \n",
       "           201879_INDL_HIST_STD_C_I  ICHIMASA KAMABOKO CO LTD  \n",
       "           221819_INDL_HIST_STD_C_I      NIPPON KANZAI CO LTD  \n",
       "...                                                       ...  \n",
       "2020-12-31 205456_INDL_HIST_STD_C_I             ONAMBA CO LTD  \n",
       "           208310_INDL_HIST_STD_C_I          SENSHUKAI CO LTD  \n",
       "           209063_INDL_HIST_STD_C_I         TOABO CORPORATION  \n",
       "           209104_INDL_HIST_STD_C_I   TOKYO OHKA KOGYO CO LTD  \n",
       "           210693_INDL_HIST_STD_C_I               PIGEON CORP  \n",
       "\n",
       "[39266 rows x 23 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compustat Global fundamentals\n",
    "_DEFAUFLT_FUNDA_FIELDS=['curcd', 'loc', 'acqmeth', 'compst', 'bspr', 'popsrc', 'auop', 'final', 'indfmt', 'consol', 'sedol',\n",
    "                        'fdate', 'pdate', 'curcdi', 'acctstd', 'iid', 'fic', 'isin', 'stalt', 'datafmt', 'costat', 'au', 'conm']\n",
    "provider_uri=\"/storage/qlib/qlib_data/wrds/comp/d_global/g_funda\"\n",
    "qlib.init(provider_uri=provider_uri)\n",
    "\n",
    "market=\"test100\"\n",
    "data_handler_config = {\n",
    "    \"start_time\": \"1987-06-30\",\n",
    "    \"end_time\": \"2020-12-31\",\n",
    "    \"fit_start_time\": \"2008-01-01\",\n",
    "    \"fit_end_time\": \"2020-12-31\",\n",
    "    \"include_fields\": _DEFAUFLT_FUNDA_FIELDS,\n",
    "    \"instruments\": market,\n",
    "}\n",
    "\n",
    "\n",
    "dataset_conifg= {\n",
    "    \"class\": \"DatasetH\",\n",
    "    \"module_path\": \"qlib.data.dataset\",\n",
    "    \"kwargs\": {\n",
    "        \"handler\": {\n",
    "            \"class\": \"WRDS\",\n",
    "            \"module_path\": \"data.wrds_handler\",\n",
    "            \"kwargs\": data_handler_config,\n",
    "        },\n",
    "        \"segments\": {\n",
    "            \"train\": (\"2008-01-01\", \"2014-12-31\"),\n",
    "            \"valid\": (\"2015-01-01\", \"2016-12-31\"),\n",
    "            \"test\": (\"2017-01-01\", \"2020-12-31\"),\n",
    "            \"all\": (\"1987-06-30\", \"2020-12-31\"),\n",
    "        },\n",
    "    },\n",
    "}\n",
    "\n",
    "dataset = init_instance_by_config(dataset_conifg)\n",
    "df=dataset.prepare(\"all\", col_set=[\"feature\", \"label\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "208479bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[4073920:MainThread](2022-03-01 13:10:13,932) INFO - qlib.Initialization - [config.py:386] - default_conf: client.\n",
      "[4073920:MainThread](2022-03-01 13:10:13,935) WARNING - qlib.Initialization - [config.py:412] - redis connection failed(host=127.0.0.1 port=6379), DiskExpressionCache and DiskDatasetCache will not be used!\n",
      "[4073920:MainThread](2022-03-01 13:10:13,936) INFO - qlib.Initialization - [__init__.py:57] - qlib successfully initialized based on client settings.\n",
      "[4073920:MainThread](2022-03-01 13:10:13,937) INFO - qlib.Initialization - [__init__.py:59] - data_path={'__DEFAULT_FREQ': PosixPath('/storage/qlib/qlib_data/wrds/comp/d_global/currency/g_exrt_mth')}\n",
      "[4073920:MainThread](2022-03-01 13:10:52,274) INFO - qlib.timer - [log.py:113] - Time cost: 22.556s | Loading data Done\n",
      "[4073920:MainThread](2022-03-01 13:10:52,277) INFO - qlib.timer - [log.py:113] - Time cost: 0.000s | fit & process data Done\n",
      "[4073920:MainThread](2022-03-01 13:10:52,278) INFO - qlib.timer - [log.py:113] - Time cost: 22.560s | Init data Done\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"20\" halign=\"left\">feature</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>exrat14m</th>\n",
       "      <th>exrat11m</th>\n",
       "      <th>fromcurm</th>\n",
       "      <th>exrat13m</th>\n",
       "      <th>exrat10m</th>\n",
       "      <th>exrat2m</th>\n",
       "      <th>exrat3m</th>\n",
       "      <th>exratm</th>\n",
       "      <th>exrat7m</th>\n",
       "      <th>exrat16m</th>\n",
       "      <th>exrat5m</th>\n",
       "      <th>exrat1m</th>\n",
       "      <th>exrat15m</th>\n",
       "      <th>exrat8m</th>\n",
       "      <th>exrat12m</th>\n",
       "      <th>exrat4m</th>\n",
       "      <th>exrat17m</th>\n",
       "      <th>exrat9m</th>\n",
       "      <th>exrat18m</th>\n",
       "      <th>exrat6m</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>datetime</th>\n",
       "      <th>instrument</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1987-06-30</th>\n",
       "      <th>AED</th>\n",
       "      <td>5.579646</td>\n",
       "      <td>5.587044</td>\n",
       "      <td>GBP</td>\n",
       "      <td>5.579420</td>\n",
       "      <td>5.600259</td>\n",
       "      <td>6.060498</td>\n",
       "      <td>6.034466</td>\n",
       "      <td>5.919040</td>\n",
       "      <td>5.740069</td>\n",
       "      <td>5.562057</td>\n",
       "      <td>5.893459</td>\n",
       "      <td>5.986990</td>\n",
       "      <td>5.572819</td>\n",
       "      <td>5.669906</td>\n",
       "      <td>5.583787</td>\n",
       "      <td>5.976899</td>\n",
       "      <td>5.540650</td>\n",
       "      <td>5.620043</td>\n",
       "      <td>5.524086</td>\n",
       "      <td>5.825870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ANG</th>\n",
       "      <td>2.729980</td>\n",
       "      <td>2.732394</td>\n",
       "      <td>GBP</td>\n",
       "      <td>2.729532</td>\n",
       "      <td>2.738281</td>\n",
       "      <td>2.953523</td>\n",
       "      <td>2.940837</td>\n",
       "      <td>2.884585</td>\n",
       "      <td>2.803704</td>\n",
       "      <td>2.721933</td>\n",
       "      <td>2.874475</td>\n",
       "      <td>2.917700</td>\n",
       "      <td>2.726937</td>\n",
       "      <td>2.770675</td>\n",
       "      <td>2.731271</td>\n",
       "      <td>2.912782</td>\n",
       "      <td>2.711695</td>\n",
       "      <td>2.747241</td>\n",
       "      <td>2.703802</td>\n",
       "      <td>2.843884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ATS</th>\n",
       "      <td>21.058121</td>\n",
       "      <td>20.489834</td>\n",
       "      <td>GBP</td>\n",
       "      <td>20.873201</td>\n",
       "      <td>20.389875</td>\n",
       "      <td>20.873663</td>\n",
       "      <td>20.833624</td>\n",
       "      <td>20.714998</td>\n",
       "      <td>20.345867</td>\n",
       "      <td>21.348837</td>\n",
       "      <td>20.510809</td>\n",
       "      <td>20.805836</td>\n",
       "      <td>21.225817</td>\n",
       "      <td>20.340284</td>\n",
       "      <td>20.668558</td>\n",
       "      <td>20.751390</td>\n",
       "      <td>21.462231</td>\n",
       "      <td>20.314529</td>\n",
       "      <td>21.611389</td>\n",
       "      <td>20.379017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AUD</th>\n",
       "      <td>2.272629</td>\n",
       "      <td>2.288751</td>\n",
       "      <td>GBP</td>\n",
       "      <td>2.288365</td>\n",
       "      <td>2.275360</td>\n",
       "      <td>2.301447</td>\n",
       "      <td>2.297211</td>\n",
       "      <td>2.240000</td>\n",
       "      <td>2.277253</td>\n",
       "      <td>2.245730</td>\n",
       "      <td>2.299004</td>\n",
       "      <td>2.269710</td>\n",
       "      <td>2.258049</td>\n",
       "      <td>2.268196</td>\n",
       "      <td>2.297307</td>\n",
       "      <td>2.302392</td>\n",
       "      <td>2.232302</td>\n",
       "      <td>2.265705</td>\n",
       "      <td>2.220981</td>\n",
       "      <td>2.294737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BEF</th>\n",
       "      <td>62.447269</td>\n",
       "      <td>60.923267</td>\n",
       "      <td>GBP</td>\n",
       "      <td>61.961746</td>\n",
       "      <td>60.628841</td>\n",
       "      <td>61.825371</td>\n",
       "      <td>61.704170</td>\n",
       "      <td>61.289982</td>\n",
       "      <td>60.486919</td>\n",
       "      <td>63.249828</td>\n",
       "      <td>60.835651</td>\n",
       "      <td>61.538006</td>\n",
       "      <td>62.896919</td>\n",
       "      <td>60.472435</td>\n",
       "      <td>61.418461</td>\n",
       "      <td>61.473961</td>\n",
       "      <td>63.565235</td>\n",
       "      <td>60.396393</td>\n",
       "      <td>63.989147</td>\n",
       "      <td>60.551086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2020-12-31</th>\n",
       "      <th>VUV</th>\n",
       "      <td>148.352325</td>\n",
       "      <td>147.771820</td>\n",
       "      <td>GBP</td>\n",
       "      <td>148.318878</td>\n",
       "      <td>147.327042</td>\n",
       "      <td>147.935989</td>\n",
       "      <td>147.522873</td>\n",
       "      <td>148.529709</td>\n",
       "      <td>146.357162</td>\n",
       "      <td>147.927902</td>\n",
       "      <td>147.070938</td>\n",
       "      <td>148.529709</td>\n",
       "      <td>148.247192</td>\n",
       "      <td>146.467133</td>\n",
       "      <td>148.018768</td>\n",
       "      <td>146.859055</td>\n",
       "      <td>147.523300</td>\n",
       "      <td>146.972153</td>\n",
       "      <td>147.240921</td>\n",
       "      <td>146.688644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XCD</th>\n",
       "      <td>3.469681</td>\n",
       "      <td>3.457523</td>\n",
       "      <td>GBP</td>\n",
       "      <td>3.469126</td>\n",
       "      <td>3.453017</td>\n",
       "      <td>3.595767</td>\n",
       "      <td>3.565229</td>\n",
       "      <td>3.625600</td>\n",
       "      <td>3.504057</td>\n",
       "      <td>3.457740</td>\n",
       "      <td>3.547586</td>\n",
       "      <td>3.625600</td>\n",
       "      <td>3.466134</td>\n",
       "      <td>3.480228</td>\n",
       "      <td>3.463460</td>\n",
       "      <td>3.547442</td>\n",
       "      <td>3.446919</td>\n",
       "      <td>3.465492</td>\n",
       "      <td>3.442400</td>\n",
       "      <td>3.525721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XOF</th>\n",
       "      <td>742.693726</td>\n",
       "      <td>735.526672</td>\n",
       "      <td>GBP</td>\n",
       "      <td>741.086609</td>\n",
       "      <td>731.379089</td>\n",
       "      <td>731.240051</td>\n",
       "      <td>728.661133</td>\n",
       "      <td>730.748413</td>\n",
       "      <td>727.077087</td>\n",
       "      <td>742.782043</td>\n",
       "      <td>727.008362</td>\n",
       "      <td>730.748413</td>\n",
       "      <td>743.223633</td>\n",
       "      <td>728.698303</td>\n",
       "      <td>738.481506</td>\n",
       "      <td>726.585754</td>\n",
       "      <td>741.179932</td>\n",
       "      <td>730.924683</td>\n",
       "      <td>740.532288</td>\n",
       "      <td>726.723083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XPF</th>\n",
       "      <td>135.420441</td>\n",
       "      <td>134.143021</td>\n",
       "      <td>GBP</td>\n",
       "      <td>135.144638</td>\n",
       "      <td>133.376007</td>\n",
       "      <td>133.861099</td>\n",
       "      <td>133.184341</td>\n",
       "      <td>134.349365</td>\n",
       "      <td>132.673508</td>\n",
       "      <td>135.408508</td>\n",
       "      <td>132.668930</td>\n",
       "      <td>134.349365</td>\n",
       "      <td>135.519348</td>\n",
       "      <td>132.938766</td>\n",
       "      <td>134.683578</td>\n",
       "      <td>132.648331</td>\n",
       "      <td>135.102097</td>\n",
       "      <td>133.329956</td>\n",
       "      <td>134.966248</td>\n",
       "      <td>132.627579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ZAR</th>\n",
       "      <td>20.743362</td>\n",
       "      <td>21.304531</td>\n",
       "      <td>GBP</td>\n",
       "      <td>20.885355</td>\n",
       "      <td>21.506851</td>\n",
       "      <td>20.507696</td>\n",
       "      <td>20.775328</td>\n",
       "      <td>20.495600</td>\n",
       "      <td>21.305471</td>\n",
       "      <td>20.446356</td>\n",
       "      <td>21.291985</td>\n",
       "      <td>20.495600</td>\n",
       "      <td>20.603403</td>\n",
       "      <td>21.438810</td>\n",
       "      <td>21.073456</td>\n",
       "      <td>20.986399</td>\n",
       "      <td>20.315239</td>\n",
       "      <td>21.608675</td>\n",
       "      <td>20.137280</td>\n",
       "      <td>21.282541</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>37587 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          feature                                   \\\n",
       "                         exrat14m    exrat11m fromcurm    exrat13m   \n",
       "datetime   instrument                                                \n",
       "1987-06-30 AED           5.579646    5.587044      GBP    5.579420   \n",
       "           ANG           2.729980    2.732394      GBP    2.729532   \n",
       "           ATS          21.058121   20.489834      GBP   20.873201   \n",
       "           AUD           2.272629    2.288751      GBP    2.288365   \n",
       "           BEF          62.447269   60.923267      GBP   61.961746   \n",
       "...                           ...         ...      ...         ...   \n",
       "2020-12-31 VUV         148.352325  147.771820      GBP  148.318878   \n",
       "           XCD           3.469681    3.457523      GBP    3.469126   \n",
       "           XOF         742.693726  735.526672      GBP  741.086609   \n",
       "           XPF         135.420441  134.143021      GBP  135.144638   \n",
       "           ZAR          20.743362   21.304531      GBP   20.885355   \n",
       "\n",
       "                                                                       \\\n",
       "                         exrat10m     exrat2m     exrat3m      exratm   \n",
       "datetime   instrument                                                   \n",
       "1987-06-30 AED           5.600259    6.060498    6.034466    5.919040   \n",
       "           ANG           2.738281    2.953523    2.940837    2.884585   \n",
       "           ATS          20.389875   20.873663   20.833624   20.714998   \n",
       "           AUD           2.275360    2.301447    2.297211    2.240000   \n",
       "           BEF          60.628841   61.825371   61.704170   61.289982   \n",
       "...                           ...         ...         ...         ...   \n",
       "2020-12-31 VUV         147.327042  147.935989  147.522873  148.529709   \n",
       "           XCD           3.453017    3.595767    3.565229    3.625600   \n",
       "           XOF         731.379089  731.240051  728.661133  730.748413   \n",
       "           XPF         133.376007  133.861099  133.184341  134.349365   \n",
       "           ZAR          21.506851   20.507696   20.775328   20.495600   \n",
       "\n",
       "                                                                       \\\n",
       "                          exrat7m    exrat16m     exrat5m     exrat1m   \n",
       "datetime   instrument                                                   \n",
       "1987-06-30 AED           5.740069    5.562057    5.893459    5.986990   \n",
       "           ANG           2.803704    2.721933    2.874475    2.917700   \n",
       "           ATS          20.345867   21.348837   20.510809   20.805836   \n",
       "           AUD           2.277253    2.245730    2.299004    2.269710   \n",
       "           BEF          60.486919   63.249828   60.835651   61.538006   \n",
       "...                           ...         ...         ...         ...   \n",
       "2020-12-31 VUV         146.357162  147.927902  147.070938  148.529709   \n",
       "           XCD           3.504057    3.457740    3.547586    3.625600   \n",
       "           XOF         727.077087  742.782043  727.008362  730.748413   \n",
       "           XPF         132.673508  135.408508  132.668930  134.349365   \n",
       "           ZAR          21.305471   20.446356   21.291985   20.495600   \n",
       "\n",
       "                                                                       \\\n",
       "                         exrat15m     exrat8m    exrat12m     exrat4m   \n",
       "datetime   instrument                                                   \n",
       "1987-06-30 AED           5.572819    5.669906    5.583787    5.976899   \n",
       "           ANG           2.726937    2.770675    2.731271    2.912782   \n",
       "           ATS          21.225817   20.340284   20.668558   20.751390   \n",
       "           AUD           2.258049    2.268196    2.297307    2.302392   \n",
       "           BEF          62.896919   60.472435   61.418461   61.473961   \n",
       "...                           ...         ...         ...         ...   \n",
       "2020-12-31 VUV         148.247192  146.467133  148.018768  146.859055   \n",
       "           XCD           3.466134    3.480228    3.463460    3.547442   \n",
       "           XOF         743.223633  728.698303  738.481506  726.585754   \n",
       "           XPF         135.519348  132.938766  134.683578  132.648331   \n",
       "           ZAR          20.603403   21.438810   21.073456   20.986399   \n",
       "\n",
       "                                                                       \n",
       "                         exrat17m     exrat9m    exrat18m     exrat6m  \n",
       "datetime   instrument                                                  \n",
       "1987-06-30 AED           5.540650    5.620043    5.524086    5.825870  \n",
       "           ANG           2.711695    2.747241    2.703802    2.843884  \n",
       "           ATS          21.462231   20.314529   21.611389   20.379017  \n",
       "           AUD           2.232302    2.265705    2.220981    2.294737  \n",
       "           BEF          63.565235   60.396393   63.989147   60.551086  \n",
       "...                           ...         ...         ...         ...  \n",
       "2020-12-31 VUV         147.523300  146.972153  147.240921  146.688644  \n",
       "           XCD           3.446919    3.465492    3.442400    3.525721  \n",
       "           XOF         741.179932  730.924683  740.532288  726.723083  \n",
       "           XPF         135.102097  133.329956  134.966248  132.627579  \n",
       "           ZAR          20.315239   21.608675   20.137280   21.282541  \n",
       "\n",
       "[37587 rows x 20 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compustat Global fundamentals\n",
    "provider_uri=\"/storage/qlib/qlib_data/wrds/comp/d_global/currency/g_exrt_mth/\"\n",
    "qlib.init(provider_uri=provider_uri)\n",
    "\n",
    "market=\"test100\"\n",
    "data_handler_config = {\n",
    "    \"start_time\": \"1987-06-30\",\n",
    "    \"end_time\": \"2020-12-31\",\n",
    "    \"fit_start_time\": \"2008-01-01\",\n",
    "    \"fit_end_time\": \"2020-12-31\",\n",
    "    \"instruments\": market,\n",
    "}\n",
    "\n",
    "\n",
    "dataset_conifg= {\n",
    "    \"class\": \"DatasetH\",\n",
    "    \"module_path\": \"qlib.data.dataset\",\n",
    "    \"kwargs\": {\n",
    "        \"handler\": {\n",
    "            \"class\": \"WRDS\",\n",
    "            \"module_path\": \"data.wrds_handler\",\n",
    "            \"kwargs\": data_handler_config,\n",
    "        },\n",
    "        \"segments\": {\n",
    "            \"train\": (\"2008-01-01\", \"2014-12-31\"),\n",
    "            \"valid\": (\"2015-01-01\", \"2016-12-31\"),\n",
    "            \"test\": (\"2017-01-01\", \"2020-12-31\"),\n",
    "            \"all\": (\"1987-06-30\", \"2020-12-31\"),\n",
    "        },\n",
    "    },\n",
    "}\n",
    "\n",
    "dataset = init_instance_by_config(dataset_conifg)\n",
    "df=dataset.prepare(\"all\", col_set=[\"feature\", \"label\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1427037c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}