Large_Scale_Transcripts.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ae96b5f4-73de-4b82-8bfc-80ff3c7024d4",
   "metadata": {},
   "source": [
    "# Large_Scale_Transcripts.ipynb\n",
    "This notebook:\n",
    "* Combines the WhisperX and Google ASR transcripts into a single csv."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1567656b-26ea-4d25-873c-183ee0a97abe",
   "metadata": {},
   "outputs": [],
   "source": [
    "import utils_general\n",
    "\n",
    "import os\n",
    "import difflib\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import json\n",
    "import string\n",
    "from urlextract import URLExtract\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "24ab0206-f41e-4149-bf3e-4d2635e69dbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Google ASR functions\n",
    "\n",
    "def clean_urls(text):\n",
    "    extractor = URLExtract()\n",
    "    urls = extractor.find_urls(text)\n",
    "    for url in urls:\n",
    "        text = text.replace(url, \"\")\n",
    "    return text\n",
    "\n",
    "# modified from https://github.com/potsawee/podcast_trec2020/blob/main/data/processor.py\n",
    "# also performs some basic cleaning\n",
    "def get_transcript_text_from_json_asr_file(json_asr_file):\n",
    "    transcript_list = []\n",
    "    with open(json_asr_file) as f:\n",
    "        transcript_dict = json.loads(f.read())\n",
    "        \n",
    "        results_list = [r for r in transcript_dict[\"results\"]]\n",
    "        last_result = results_list[-1]\n",
    "        \n",
    "        for word_dict in last_result[\"alternatives\"][0][\"words\"]:\n",
    "            endTime = float(word_dict[\"endTime\"].replace(\"s\",\"\"))\n",
    "            if endTime <= (60 * 10):  # truncates the podcasts to 10 minutes\n",
    "                transcript_list.append(word_dict[\"word\"])\n",
    "        \n",
    "        transcript_string = \" \".join(transcript_list)\n",
    "        \n",
    "        # clean the transcripts the same way as the descriptions\n",
    "        transcript_string = clean_urls(transcript_string)\n",
    "        transcript_string = transcript_string.encode(\"ascii\", \"ignore\").decode()\n",
    "        \n",
    "        if transcript_string != \"\" and transcript_string[-1] not in string.punctuation:\n",
    "            transcript_string += \".\"\n",
    "        \n",
    "        return transcript_string\n",
    "\n",
    "# WhisperX functions\n",
    "def get_ascii_text(dictionary):\n",
    "    text = \"\"\n",
    "    \n",
    "    # read text from json object\n",
    "    for t in dictionary[\"segments\"]:\n",
    "        text += t[\"text\"]\n",
    "    text = text.strip()\n",
    "    \n",
    "    # only allow ascii characters\n",
    "    text = text.encode(\"ascii\", \"ignore\").decode()\n",
    "        \n",
    "    return text\n",
    "\n",
    "df = pd.read_csv(\"./csv/df.csv\")\n",
    "df = df.rename(columns={\"transcript\": \"WhisperX\"})\n",
    "\n",
    "# for the files\n",
    "pbar = tqdm(total=len(df))\n",
    "for index, row in df.iterrows():\n",
    "    \n",
    "    # read in the GoogleASR transcriptions\n",
    "    show_filename_prefix = row[\"show_filename_prefix\"]\n",
    "    episode_filename_prefix = row[\"episode_filename_prefix\"]\n",
    "    dir1 = show_filename_prefix.split(\"_\")[1][0].upper()\n",
    "    dir2 = show_filename_prefix.split(\"_\")[1][1].upper()\n",
    "    json_asr_file = os.path.join(\"/data2/maria/Spotify-Podcasts/podcasts-no-audio-13GB/spotify-podcasts-2020/podcasts-transcripts\", dir1, dir2, show_filename_prefix, episode_filename_prefix+\".json\")\n",
    "\n",
    "    df.loc[index, \"GoogleASR\"] = get_transcript_text_from_json_asr_file(json_asr_file)\n",
    "    \n",
    "    # update progress bar\n",
    "    pbar.update(1)\n",
    "\n",
    "display(df)\n",
    "\n",
    "df.to_csv(\"./csv/large_scale_texts.csv\", header=True)\n",
    "\n",
    "def split_dataframe(big_df, n_parts):\n",
    "\n",
    "    # Split the indices into n_parts parts\n",
    "    indices = np.array(big_df.index)\n",
    "    parts_indices = np.array_split(indices, n_parts)\n",
    "\n",
    "    # Iterate through the parts and create DataFrames\n",
    "    split_dfs = []\n",
    "    for part_indices in parts_indices:\n",
    "        part_df = big_df.loc[part_indices]\n",
    "        split_dfs.append(part_df)\n",
    "\n",
    "    return split_dfs\n",
    "\n",
    "\n",
    "big_df = pd.read_csv(\"./csv/large_scale_texts.csv\")\n",
    "\n",
    "n_parts = 2\n",
    "split_dfs = split_dataframe(big_df, n_parts)\n",
    "\n",
    "lens = []\n",
    "for index, df in enumerate(split_dfs):\n",
    "    lens.append(len(df))\n",
    "    csv_path = f\"./csv/large_scale_texts-{index}.csv\"\n",
    "    utils_general.delete_file_if_already_exists(csv_path)\n",
    "    df.to_csv(csv_path, header=True)\n",
    "    \n",
    "assert sum(lens) == len(big_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87b98fce-868c-45e2-bb0a-f7ceac736229",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "2022_10_3",
   "language": "python",
   "name": "2022_10_3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}