54点训练任务生成及后处理.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/octusr3/anaconda3/lib/python3.7/site-packages/pandas/compat/_optional.py:138: UserWarning: Pandas requires version '2.7.0' or newer of 'numexpr' (version '2.6.8' currently installed).\n",
      "  warnings.warn(msg, UserWarning)\n"
     ]
    },
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'Imagen' from 'PIL' (/home/octusr3/anaconda3/lib/python3.7/site-packages/PIL/__init__.py)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-de7ad5d33711>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mseaborn\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msns\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mhashlib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mPIL\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mImagen\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mdatastatics\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0moct_Utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: cannot import name 'Imagen' from 'PIL' (/home/octusr3/anaconda3/lib/python3.7/site-packages/PIL/__init__.py)"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import json\n",
    "import cv2\n",
    "import os\n",
    "import re\n",
    "import pickle\n",
    "import math\n",
    "from pathlib import *\n",
    "import multiprocessing\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from collections import *\n",
    "from itertools import *\n",
    "from functools import *\n",
    "from sklearn.metrics import *\n",
    "from scipy.stats import *\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import hashlib\n",
    "from PIL import Imagen\n",
    "from datastatics import *\n",
    "from oct_Utils import *\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bupt=pd.read_csv('data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_path</th>\n",
       "      <th>source</th>\n",
       "      <th>pid</th>\n",
       "      <th>dates</th>\n",
       "      <th>eye</th>\n",
       "      <th>pos</th>\n",
       "      <th>num</th>\n",
       "      <th>pd_prob</th>\n",
       "      <th>dataset</th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20210730-中山眼科-OCT转VF/CASIA数据汇总-new/CS-199/2019...</td>\n",
       "      <td>CASIA数据汇总-new</td>\n",
       "      <td>CS-199</td>\n",
       "      <td>2019-06-12</td>\n",
       "      <td>OD</td>\n",
       "      <td>disc</td>\n",
       "      <td>[[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...</td>\n",
       "      <td>[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 1...</td>\n",
       "      <td>0</td>\n",
       "      <td>[25.562647   26.01373    25.540567   25.702322...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20210730-中山眼科-OCT转VF/CASIA数据汇总-new/CS-199/2019...</td>\n",
       "      <td>CASIA数据汇总-new</td>\n",
       "      <td>CS-199</td>\n",
       "      <td>2019-06-12</td>\n",
       "      <td>OS</td>\n",
       "      <td>disc</td>\n",
       "      <td>[[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...</td>\n",
       "      <td>[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 1...</td>\n",
       "      <td>0</td>\n",
       "      <td>[25.731358  26.288353  25.760395  25.932634  2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20210730-中山眼科-OCT转VF/CASIA数据汇总-new/CS-201/2019...</td>\n",
       "      <td>CASIA数据汇总-new</td>\n",
       "      <td>CS-201</td>\n",
       "      <td>2019-06-13</td>\n",
       "      <td>OD</td>\n",
       "      <td>disc</td>\n",
       "      <td>[[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...</td>\n",
       "      <td>[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 3...</td>\n",
       "      <td>0</td>\n",
       "      <td>[26.397497  26.833364  26.385307  26.588284  2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20210730-中山眼科-OCT转VF/CASIA数据汇总-new/CS-201/2019...</td>\n",
       "      <td>CASIA数据汇总-new</td>\n",
       "      <td>CS-201</td>\n",
       "      <td>2019-06-13</td>\n",
       "      <td>OS</td>\n",
       "      <td>disc</td>\n",
       "      <td>[[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...</td>\n",
       "      <td>[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 1...</td>\n",
       "      <td>0</td>\n",
       "      <td>[27.085466   27.53569    27.00123    27.221846...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20210730-中山眼科-OCT转VF/CASIA数据汇总-new/CS-204/2019...</td>\n",
       "      <td>CASIA数据汇总-new</td>\n",
       "      <td>CS-204</td>\n",
       "      <td>2019-06-13</td>\n",
       "      <td>OD</td>\n",
       "      <td>disc</td>\n",
       "      <td>[[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...</td>\n",
       "      <td>[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 1...</td>\n",
       "      <td>0</td>\n",
       "      <td>[26.422417  26.930092  26.382294  26.551111  2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13387</th>\n",
       "      <td>20210820-sysu-oct/MAGIC4/MAGIC2238/20210628/OS...</td>\n",
       "      <td>MAGIC4</td>\n",
       "      <td>MAGIC2238</td>\n",
       "      <td>2021-06-28</td>\n",
       "      <td>OS</td>\n",
       "      <td>disc</td>\n",
       "      <td>[[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...</td>\n",
       "      <td>[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 1...</td>\n",
       "      <td>4</td>\n",
       "      <td>[25.635014  25.887339  25.271215  25.449888  2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13388</th>\n",
       "      <td>20210820-sysu-oct/MAGIC4/MAGIC2242/20210713/OD...</td>\n",
       "      <td>MAGIC4</td>\n",
       "      <td>MAGIC2242</td>\n",
       "      <td>2021-07-13</td>\n",
       "      <td>OD</td>\n",
       "      <td>disc</td>\n",
       "      <td>[[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...</td>\n",
       "      <td>[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 4...</td>\n",
       "      <td>4</td>\n",
       "      <td>[26.36651   26.712744  26.094995  26.401798  2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13389</th>\n",
       "      <td>20210820-sysu-oct/MAGIC4/MAGIC2242/20210713/OD...</td>\n",
       "      <td>MAGIC4</td>\n",
       "      <td>MAGIC2242</td>\n",
       "      <td>2021-07-13</td>\n",
       "      <td>OD</td>\n",
       "      <td>disc</td>\n",
       "      <td>[[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...</td>\n",
       "      <td>[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 4...</td>\n",
       "      <td>4</td>\n",
       "      <td>[25.682802   26.011946   25.458757   25.721516...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13390</th>\n",
       "      <td>20210820-sysu-oct/MAGIC4/MAGIC2242/20210713/OS...</td>\n",
       "      <td>MAGIC4</td>\n",
       "      <td>MAGIC2242</td>\n",
       "      <td>2021-07-13</td>\n",
       "      <td>OS</td>\n",
       "      <td>disc</td>\n",
       "      <td>[[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...</td>\n",
       "      <td>[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 3...</td>\n",
       "      <td>4</td>\n",
       "      <td>[26.564531  26.755457  26.124134  26.330153  2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13391</th>\n",
       "      <td>20210820-sysu-oct/MAGIC4/MAGIC2242/20210713/OS...</td>\n",
       "      <td>MAGIC4</td>\n",
       "      <td>MAGIC2242</td>\n",
       "      <td>2021-07-13</td>\n",
       "      <td>OS</td>\n",
       "      <td>disc</td>\n",
       "      <td>[[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...</td>\n",
       "      <td>[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 3...</td>\n",
       "      <td>4</td>\n",
       "      <td>[25.785372   26.111225   25.474897   25.700928...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13392 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              image_path         source  \\\n",
       "0      20210730-中山眼科-OCT转VF/CASIA数据汇总-new/CS-199/2019...  CASIA数据汇总-new   \n",
       "1      20210730-中山眼科-OCT转VF/CASIA数据汇总-new/CS-199/2019...  CASIA数据汇总-new   \n",
       "2      20210730-中山眼科-OCT转VF/CASIA数据汇总-new/CS-201/2019...  CASIA数据汇总-new   \n",
       "3      20210730-中山眼科-OCT转VF/CASIA数据汇总-new/CS-201/2019...  CASIA数据汇总-new   \n",
       "4      20210730-中山眼科-OCT转VF/CASIA数据汇总-new/CS-204/2019...  CASIA数据汇总-new   \n",
       "...                                                  ...            ...   \n",
       "13387  20210820-sysu-oct/MAGIC4/MAGIC2238/20210628/OS...         MAGIC4   \n",
       "13388  20210820-sysu-oct/MAGIC4/MAGIC2242/20210713/OD...         MAGIC4   \n",
       "13389  20210820-sysu-oct/MAGIC4/MAGIC2242/20210713/OD...         MAGIC4   \n",
       "13390  20210820-sysu-oct/MAGIC4/MAGIC2242/20210713/OS...         MAGIC4   \n",
       "13391  20210820-sysu-oct/MAGIC4/MAGIC2242/20210713/OS...         MAGIC4   \n",
       "\n",
       "             pid       dates eye   pos  \\\n",
       "0         CS-199  2019-06-12  OD  disc   \n",
       "1         CS-199  2019-06-12  OS  disc   \n",
       "2         CS-201  2019-06-13  OD  disc   \n",
       "3         CS-201  2019-06-13  OS  disc   \n",
       "4         CS-204  2019-06-13  OD  disc   \n",
       "...          ...         ...  ..   ...   \n",
       "13387  MAGIC2238  2021-06-28  OS  disc   \n",
       "13388  MAGIC2242  2021-07-13  OD  disc   \n",
       "13389  MAGIC2242  2021-07-13  OD  disc   \n",
       "13390  MAGIC2242  2021-07-13  OS  disc   \n",
       "13391  MAGIC2242  2021-07-13  OS  disc   \n",
       "\n",
       "                                                     num  \\\n",
       "0      [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...   \n",
       "1      [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...   \n",
       "2      [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...   \n",
       "3      [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...   \n",
       "4      [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...   \n",
       "...                                                  ...   \n",
       "13387  [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...   \n",
       "13388  [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...   \n",
       "13389  [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...   \n",
       "13390  [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...   \n",
       "13391  [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\\n [...   \n",
       "\n",
       "                                                 pd_prob  dataset  \\\n",
       "0      [[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 1...        0   \n",
       "1      [[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 1...        0   \n",
       "2      [[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 3...        0   \n",
       "3      [[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 1...        0   \n",
       "4      [[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 1...        0   \n",
       "...                                                  ...      ...   \n",
       "13387  [[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 1...        4   \n",
       "13388  [[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 4...        4   \n",
       "13389  [[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 4...        4   \n",
       "13390  [[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 3...        4   \n",
       "13391  [[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\\n [0. 0. 0. 3...        4   \n",
       "\n",
       "                                                    pred  \n",
       "0      [25.562647   26.01373    25.540567   25.702322...  \n",
       "1      [25.731358  26.288353  25.760395  25.932634  2...  \n",
       "2      [26.397497  26.833364  26.385307  26.588284  2...  \n",
       "3      [27.085466   27.53569    27.00123    27.221846...  \n",
       "4      [26.422417  26.930092  26.382294  26.551111  2...  \n",
       "...                                                  ...  \n",
       "13387  [25.635014  25.887339  25.271215  25.449888  2...  \n",
       "13388  [26.36651   26.712744  26.094995  26.401798  2...  \n",
       "13389  [25.682802   26.011946   25.458757   25.721516...  \n",
       "13390  [26.564531  26.755457  26.124134  26.330153  2...  \n",
       "13391  [25.785372   26.111225   25.474897   25.700928...  \n",
       "\n",
       "[13392 rows x 10 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bupt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "map_matrix='''[ [0  0  0  0  0  0  0  0  0  0]\n",
    "                [0  0  0  46 44 43 42 0  0  0]\n",
    "                [0  0  46 45 44 43 41 43 0  0]\n",
    "                [0  45 46 45 45 45 42 42 41 0]\n",
    "                [46 46 47 46 50 50 50 3  41 0]\n",
    "                [9  9  9  6  8  6  7  3  16 0]\n",
    "                [0  10 8  7  6  8  9  13 17 0]\n",
    "                [0  0  8  8  10 10 12 15 0  0]\n",
    "                [0  0  0  9  10 11 14 0  0  0]\n",
    "                [0  0  0  0  0  0  0  0  0  0]]'''#为了凑54个点，给两个视野盲区的点都随便指向了一个2\n",
    "\n",
    "\n",
    "def str_to_np_array(row, dtype=np.float):\n",
    "    row = row.replace('[', '').replace(']', '')\n",
    "    row = row.split(' ')\n",
    "    row = filter(lambda x: x != '', row)\n",
    "    row = map(lambda x: dtype(x), row)\n",
    "    row = np.array(list(row))\n",
    "    return row\n",
    "\n",
    "def str_to_np_mat(s, dtype=np.float):\n",
    "    s = s[1:-1]\n",
    "    rows = s.split('\\n')\n",
    "    for index, row in enumerate(rows):\n",
    "        rows[index] = str_to_np_array(row, dtype)\n",
    "    rows = np.stack(rows, axis=0)\n",
    "    return rows\n",
    "\n",
    "\n",
    "def calculate_position(arr):\n",
    "\t#拼接数组函数\n",
    "    array_num = list(itertools.chain.from_iterable(arr))\n",
    "    array_num = np.array(array_num)\n",
    "    key = np.unique(array_num)\n",
    "    result = {}\n",
    "    for k in key:\n",
    "        index=np.argwhere(arr == k)#返回所有符合条件的索引值，更多的方法和类参数需要看\n",
    "        v=[tuple(index[i]) for i in range(0,len(index))]\n",
    "        result[k] = v\n",
    "    result = {key:val for key, val in result.items() if  len(val)<9}\n",
    "\n",
    "    valid_point=[]\n",
    "    for key,value in result.items():\n",
    "        valid_point+=value\n",
    "\n",
    "    return result,valid_point\n",
    "\n",
    "num_mat=str_to_np_mat(map_matrix)\n",
    "countNums,valid_point=calculate_position(num_mat)#之前statics的时候，统计的数据是用的repeat的重复数据，这次要重新处理，尽量保证模块化，工程化\n",
    "point_to_image_slice=reverse_dict(countNums)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.read_csv('data.csv')#\n",
    "data['np_mat_num']=data['num'].apply(str_to_np_mat)\n",
    "data['np_mat_num']=data['np_mat_num'].mask(data['eye']=='OD',data['np_mat_num'].apply(lambda x:np.flip(x,axis=1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.series.Series"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(data[\"np_mat_num\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.read_csv('data.csv')#\n",
    "data['np_mat_num']=data['num'].apply(str_to_np_mat)\n",
    "data['np_mat_num']=data['np_mat_num'].mask(data['eye']=='OD',data['np_mat_num'].apply(lambda x:np.flip(x,axis=1)))#all_num已经翻转过，所以数据是正常的,这里相当于\n",
    "#把label直接翻转好了，后续根据input直接做就可以了\n",
    "all_num=data['np_mat_num']\n",
    "\n",
    "def split_np_matrix(all_num,data:pd.DataFrame,valid_point):\n",
    "    res=np.zeros((1,100))\n",
    "    for num in all_num:\n",
    "        res=np.concatenate((res,np.expand_dims(num.flatten(),0)),axis=0)\n",
    "    res=res[1:]\n",
    "    for i in range(0,res.shape[1]):\n",
    "        if (i//10,i%10) in valid_point:\n",
    "            data[f'({i//10},{(i)%10})']=res[:,i]\n",
    "    return data\n",
    "    \n",
    "def get_slice(temp,point_to_image_slice):\n",
    "    return  point_to_image_slice[temp]\n",
    "    \n",
    "new_data=split_np_matrix(all_num,data,valid_point)#分离np_matrix数组，实际上可用explode进行处理\n",
    "new_data.to_csv('new_data.csv')#这里用excel preview看数据就出现了19.0，但实际上通过调用columns属性就可以得出其实数据属性列是正常的\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_csv(data):\n",
    "    data=pd.read_csv(data)\n",
    "    column_name=data.columns.values.tolist()\n",
    "    data=data.drop(['Unnamed: 0','np_mat_num','pid','source','pos','num','pd_prob','np_mat_num','dates'],axis=1)\n",
    "    col=data.columns.values.tolist()\n",
    "    index=[i%len(data) for i in range(0,len(data)*54)]\n",
    "    data=pd.melt(data,id_vars=col[0:3],value_vars=col[3:],var_name='position',value_name='value')\n",
    "    data=data.set_index(pd.Index(index)).sort_index()\n",
    "    #res.to_csv('one_input_data')\n",
    "    data['A'],data['B'],data['C']=data['image_path'].str.split('.',2).str\n",
    "    data['slice']=data['position'].apply(get_slice,args=(point_to_image_slice,))\n",
    "    data['slice']=data['slice'].map(lambda x:str(x))\n",
    "    data['image_path']=data['A'].str.cat(data['B'],sep='.').str.cat(data['slice'],sep='_').str.cat(data['C'],sep='.')\n",
    "    data.drop(['A','B','C'],axis=1,inplace=True)\n",
    "    data.to_csv('one_input_data.csv')\n",
    "\n",
    "generate_csv('new_data.csv') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('data.csv')#用来生成outpu下的文件夹,生成训练任务\n",
    "id_base = 'pid'\n",
    "date = '1005'\n",
    "label_type = 'num'\n",
    "\n",
    "for model, size, pos in product(['r50'], ['380'], ['disc', 'macula']):\n",
    "    exp_name = f'{date}/{pos}_{label_type}_{model}_{size}'\n",
    "    df_tmp = df.copy()\n",
    "    df_tmp = df_tmp[df_tmp['pos'] == pos]\n",
    "    df_tmp['dataset'] = df_tmp[id_base].map(to_dataset_mapping(df_tmp[id_base].tolist(), 5))\n",
    "    ensure_path(f'output/{exp_name}/exp-{id_base}/tasks')\n",
    "    df_tmp.to_csv(f'output/{exp_name}/exp-{id_base}/tasks/data.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "以下为数据后处理部分，用以生成原来形式的数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def back_treat(res):\n",
    "    res=pd.read_csv('res.csv')\n",
    "    res['A'],res['B'],res['C']=res['image_path'].str.split('.',2).str\n",
    "    res['image_path']=res['A'].str.cat(res['B'].str[0:3],sep='.').str.cat(res['C'],sep='.')\n",
    "    res.drop(['A','B','C'],axis=1,inplace=True)\n",
    "    data=pd.DataFrame(res.set_index(['image_path','position'])['pred'])\n",
    "    df1=data.pivot_table(index='image_path',values='pred',columns='position')\n",
    "    return df1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/octusr3/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: FutureWarning: Columnar iteration over characters will be deprecated in future releases.\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "res=pd.read_csv('res.csv')\n",
    "res['A'],res['B'],res['C']=res['image_path'].str.split('.',2).str\n",
    "res['image_path']=res['A'].str.cat(res['B'].str[0:3],sep='.').str.cat(res['C'],sep='.')\n",
    "res.drop(['A','B','C'],axis=1,inplace=True)\n",
    "data=pd.DataFrame(res.set_index(['image_path','position'])['pred'])\n",
    "df1=data.pivot_table(index='image_path',values='pred',columns='position')\n",
    "original_data=df1.copy().reset_index()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def concat_num(res,orignal_data):#用来将54个数字拼成数组\n",
    "    temp=[]\n",
    "    for i in res.values:\n",
    "        temp.append(list(i)) \n",
    "    original_data['result']=temp\n",
    "    return original_data\n",
    "\n",
    "def convert_num_pred_to_str(tmp):\n",
    "    cos=np.array(tmp).reshape(9,6)\n",
    "    tep=[str(i).replace(' ','    ').replace('[','').replace(']','') for i in cos]\n",
    "    re=''.join(x+' ' for x in tep)#不能 加\\n,因为刚刚打印显示出来的字符串显示有'\\n',并不是字符串里有的，而是就那么显示的\n",
    "    return '['+re+']'\n",
    "    #这里加' '是为了区分两个数字，不至于连到一起\n",
    "\n",
    "original_data=concat_num(df1,original_data)\n",
    "original_data=pd.DataFrame(original_data,columns=['image_path','result'])\n",
    "original_data['pred']=original_data['result'].apply(convert_num_pred_to_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ripe_data=pd.read_csv('data.csv')\n",
    "temp=ripe_data.drop_duplicates(subset=['image_path'])\n",
    "temp=temp.drop(['pred'],axis=1)\n",
    "res=temp.merge(original_data,how='right',on='image_path')\n",
    "res.drop(['result'],axis=1,inplace=True)\n",
    "res.to_csv('bupt.csv',index=False)#生成最后结果的代码\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10598, 10)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coret=pd.read_csv('bupt.csv')\n",
    "coret.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "经过测试以及论证，包括修改代码，发现在原始数据data.csv文件中，对一列数据进行统计  fds是9321条，fda是 4071条  正常\n",
    "目前现有的最后做出来的结果数据 缺损数据  fda是3033   fds 7595 共10598条，无论怎么都对不上,后来经过测试数据发现原本的data.csv文件中有完全一模一样的两条数据\n",
    "原始数据没有drop掉重复项，在直接预测原始的54个点的时候没有问题，但是分开做54点的时候，使用pivot，同一个元素就会有键覆盖的问题，最后导致数据总体数目不一样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "a=df1.reset_index()['image_path']#数据检测异常代码,进行数据检查\n",
    "bupt=pd.read_csv('data.csv')['image_path']\n",
    "d={'col1':a,'col2':bupt}\n",
    "com=pd.DataFrame(data=d)\n",
    "com.to_csv('feaf.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'20210730-中山眼科-OCT转VF/CASIA数据汇总-new/CRC-009-143/OS/OCT-后/3D Disc/69627.fds_50.jpg'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['image_path'][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "correct_data=pd.read_csv('data.csv')\n",
    "temp=correct_data[correct_data['image_path']=='20210730-中山眼科-OCT转VF/CASIA数据汇总-new/CRC-009-143/OS/OCT-后/3D Disc/69627.fds.jpg']"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "623a9f8af0be4123d72a49457fdd65b442f8fe6d5b5109249813d94319742a73"
  },
  "kernelspec": {
   "display_name": "Python 3.7.2 64-bit ('base': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}