data/process.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tqdm\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = [os.path.join('meta', x) for x in os.listdir('meta')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 27638/27638 [00:08<00:00, 3233.21it/s]\n"
     ]
    }
   ],
   "source": [
    "grid = []\n",
    "nutrition_desc = dict()\n",
    "\n",
    "for d in tqdm.tqdm(data):\n",
    "    row = dict()\n",
    "\n",
    "    with open(d) as file:\n",
    "        obj = json.load(file)\n",
    "        row['src'] = obj['source'].get('sourceRecipeUrl', None)\n",
    "        row['ingredients'] = ', '.join(obj['ingredientLines'])\n",
    "        \n",
    "        nutrition = obj['nutritionEstimates']\n",
    "        for n in nutrition:\n",
    "            nutrition_desc[n['attribute']] = {\n",
    "                'unit': n['unit']['plural'],\n",
    "                'description': n['description'],\n",
    "            }\n",
    "            row[n['attribute']] = n['value']\n",
    "    \n",
    "    grid.append(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data=grid)\n",
    "df.to_csv('data.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>ingredients</th>\n",
       "      <th>sugar</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://www.myrecipes.com/recipe/stuffed-cherry...</td>\n",
       "      <td>2 pints cherry tomatoes, 1 avocado, peeled and...</td>\n",
       "      <td>3.49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://paleomg.com/simple-baked-halibut-with-b...</td>\n",
       "      <td>1 pound halibut (cut into 2 eight ounce pieces...</td>\n",
       "      <td>0.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://allrecipes.com/Recipe/honey-whole-wheat...</td>\n",
       "      <td>1 cup water (70 degrees to 80 degrees), 1/4 cu...</td>\n",
       "      <td>2.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://eatwithmarco.com/en/pasta-alla-norma/</td>\n",
       "      <td>3 melanzane / 3 eggplants, 1 spicchio d’aglio ...</td>\n",
       "      <td>3.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://www.myrecipes.com/recipe/plum-upside-do...</td>\n",
       "      <td>2 teaspoons butter, 6 large red plums, pitted ...</td>\n",
       "      <td>38.17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 src   \n",
       "0  http://www.myrecipes.com/recipe/stuffed-cherry...  \\\n",
       "1  http://paleomg.com/simple-baked-halibut-with-b...   \n",
       "2  http://allrecipes.com/Recipe/honey-whole-wheat...   \n",
       "3       http://eatwithmarco.com/en/pasta-alla-norma/   \n",
       "4  http://www.myrecipes.com/recipe/plum-upside-do...   \n",
       "\n",
       "                                         ingredients  sugar  \n",
       "0  2 pints cherry tomatoes, 1 avocado, peeled and...   3.49  \n",
       "1  1 pound halibut (cut into 2 eight ounce pieces...   0.17  \n",
       "2  1 cup water (70 degrees to 80 degrees), 1/4 cu...   2.15  \n",
       "3  3 melanzane / 3 eggplants, 1 spicchio d’aglio ...   3.78  \n",
       "4  2 teaspoons butter, 6 large red plums, pitted ...  38.17  "
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(data=grid)\n",
    "df = df[df['SUGAR'].notna()]\n",
    "df.rename(columns = {'SUGAR':'sugar'}, inplace = True)\n",
    "df = df[['src', 'ingredients', 'sugar']]\n",
    "df.fillna(0, inplace=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10.85810766787474, 13.3627190349059)"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[df['sugar'] < 100]\n",
    "label = df['sugar']\n",
    "label = np.array(label)\n",
    "mean = label.mean()\n",
    "std = label.std()\n",
    "mean, std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-2.8934427485446114e-17, 0.9999999999999999)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label = (label - mean) / std\n",
    "label.mean(), label.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/t8/n0wxg8y925qg6fcsjxz2zgqr0000gn/T/ipykernel_91137/1060462324.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['sugar'] = label\n"
     ]
    }
   ],
   "source": [
    "df['sugar'] = label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>ingredients</th>\n",
       "      <th>sugar</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://www.myrecipes.com/recipe/stuffed-cherry...</td>\n",
       "      <td>2 pints cherry tomatoes, 1 avocado, peeled and...</td>\n",
       "      <td>-0.551393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://paleomg.com/simple-baked-halibut-with-b...</td>\n",
       "      <td>1 pound halibut (cut into 2 eight ounce pieces...</td>\n",
       "      <td>-0.799845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://allrecipes.com/Recipe/honey-whole-wheat...</td>\n",
       "      <td>1 cup water (70 degrees to 80 degrees), 1/4 cu...</td>\n",
       "      <td>-0.651672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://eatwithmarco.com/en/pasta-alla-norma/</td>\n",
       "      <td>3 melanzane / 3 eggplants, 1 spicchio d’aglio ...</td>\n",
       "      <td>-0.529691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://www.myrecipes.com/recipe/plum-upside-do...</td>\n",
       "      <td>2 teaspoons butter, 6 large red plums, pitted ...</td>\n",
       "      <td>2.043887</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 src   \n",
       "0  http://www.myrecipes.com/recipe/stuffed-cherry...  \\\n",
       "1  http://paleomg.com/simple-baked-halibut-with-b...   \n",
       "2  http://allrecipes.com/Recipe/honey-whole-wheat...   \n",
       "3       http://eatwithmarco.com/en/pasta-alla-norma/   \n",
       "4  http://www.myrecipes.com/recipe/plum-upside-do...   \n",
       "\n",
       "                                         ingredients     sugar  \n",
       "0  2 pints cherry tomatoes, 1 avocado, peeled and... -0.551393  \n",
       "1  1 pound halibut (cut into 2 eight ounce pieces... -0.799845  \n",
       "2  1 cup water (70 degrees to 80 degrees), 1/4 cu... -0.651672  \n",
       "3  3 melanzane / 3 eggplants, 1 spicchio d’aglio ... -0.529691  \n",
       "4  2 teaspoons butter, 6 large red plums, pitted ...  2.043887  "
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('sugar.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89f468f654a146b098784b509395020f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "\n",
    "notebook_login()#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"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset csv (/Users/ziqin/.cache/huggingface/datasets/csv/default-11981611f38d66e1/0.0.0/6954658bab30a358235fa864b05cf819af0e179325c740e4bc853bcc7ec513e1)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b4a615c38c364653846cd8e900e80cc8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset = load_dataset(\"csv\", data_files=\"sugar.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = dataset['train'].shuffle(seed=42).train_test_split(test_size=0.08)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['src', 'ingredients', 'sugar'],\n",
       "        num_rows: 22592\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['src', 'ingredients', 'sugar'],\n",
       "        num_rows: 1965\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Pushing split train to the Hub.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61ea1df3967d4cc19f3e867752c5d56a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Pushing dataset shards to the dataset hub:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "594e33aeb4c6454fa1c85a559d7f3f3b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/23 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9203f52ecb454f11a0d8ec04f068a482",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload 1 LFS files:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6099d84c0af1461eb81597076472e48e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Deleting unused files from dataset repository:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Pushing split test to the Hub.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5945961bc88546a7aecad7d6a9d2b158",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Pushing dataset shards to the dataset hub:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6cef1f3b5f474156a711e45fdcdae8a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/2 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34f13dba97a94a4684cfb9067522338f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload 1 LFS files:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7600746cfe84219b8e2752c048d898e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Deleting unused files from dataset repository:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a3303c0eaa06484fa703d915af05b355",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading metadata:   0%|          | 0.00/527 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset.push_to_hub(\"ziq/ingredient_to_sugar_level\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}