Large_Scale_Filtering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c28f0e55-ec43-4ce6-8aa1-c1fb7eb45f55",
   "metadata": {},
   "source": [
    "# Large_Scale_Filtering.ipynb\n",
    "This notebook performs:\n",
    "* Basic filtering on the ~100k podcasts.\n",
    "* Adds basic features to the df. \n",
    "* Writes in the ascii WhisperX transcripts into the main df. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f3ea9fba-f1fa-4372-ab38-003eb401b1de",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 105360/105360 [13:30<00:00, 128.00it/s]"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import json\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "\n",
    "import utils_general\n",
    "\n",
    "def get_ascii_text(dictionary):\n",
    "    text = \"\"\n",
    "    \n",
    "    # read text from json object\n",
    "    for t in dictionary[\"segments\"]:\n",
    "        text += t[\"text\"]\n",
    "    text = text.strip()\n",
    "    \n",
    "    # only allow ascii characters\n",
    "    text = text.encode(\"ascii\", \"ignore\").decode()\n",
    "        \n",
    "    return text\n",
    "\n",
    "# create df based off of metadata df\n",
    "df = pd.read_csv(utils_general.PATH_TO_TRAIN_DF, sep=\"\\t\")\n",
    "\n",
    "# filter df to only include podcasts at least 10 minutes long\n",
    "df = df[df[\"duration\"] >= 10.0]\n",
    "\n",
    "# recreate language column (setup for languages based on whisperx, not on existing language id)\n",
    "df = df.drop(\"language\", axis=1)\n",
    "df[\"language\"] = \"\"\n",
    "\n",
    "# create column for whisperxtranscripts\n",
    "df[\"transcript\"] = \"\"\n",
    "\n",
    "# iterate through json files to add to df\n",
    "pbar = tqdm(total=105360)\n",
    "for root, dirs, files in os.walk(\"/data1/maria/Spotify-Podcasts/train-10min-whisperx-dir\"):\n",
    "    if files:\n",
    "        for file in files:\n",
    "            if file == \"transcript.json\":\n",
    "                \n",
    "                full_filepath = os.path.join(root,\"transcript.json\")\n",
    "                episode_id = os.path.split(root)[-1]\n",
    "                with open(full_filepath) as f:\n",
    "                    dictionary = json.loads(f.read())\n",
    "\n",
    "                df.loc[df[\"episode_filename_prefix\"] == episode_id, \"language\"] = dictionary[\"language\"]\n",
    "                df.loc[df[\"episode_filename_prefix\"] == episode_id, \"transcript\"] = get_ascii_text(dictionary)\n",
    "                \n",
    "                pbar.update(1)\n",
    "\n",
    "# only allow english transcripts\n",
    "df = df[df[\"language\"] == \"en\"]\n",
    "\n",
    "# calculate number of words in transcripts\n",
    "df[\"transcript_length\"] = -1\n",
    "df[\"transcript\"] = df[\"transcript\"].fillna(\"\")\n",
    "for index, row in df.iterrows():\n",
    "    # handle special case of empty string\n",
    "    if row[\"transcript\"] != \"\":\n",
    "        num_words = len(row[\"transcript\"].split(\" \"))\n",
    "    else:\n",
    "        num_words = 0\n",
    "    # write in the number of words\n",
    "    df.loc[index, \"transcript_length\"] = num_words\n",
    "\n",
    "# save dfs with zero words for examination\n",
    "zero_words_df = df[df[\"transcript_length\"] == 0]\n",
    "zero_words_df.to_csv(\"./csv/zero_words_df.csv\", header=True)\n",
    "\n",
    "# filter out transcripts with zero words\n",
    "df = df[df[\"transcript_length\"] != 0]\n",
    "\n",
    "# save dfs with less than 10 words for examination\n",
    "less_than_10_words_df = df[df[\"transcript_length\"] < 10]\n",
    "less_than_10_words_df.to_csv(\"./csv/less_than_10_words_df.csv\", header=True)\n",
    "\n",
    "# filter out transcripts with less than 10 words\n",
    "df = df[df[\"transcript_length\"] > 10]\n",
    "\n",
    "# save df\n",
    "df.to_csv(\"./csv/df.csv\", header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb3b100b-4f3b-4734-8cb1-961ba4df1c50",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:general]",
   "language": "python",
   "name": "conda-env-general-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}