word-counter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Word counter using PySpark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "import pyspark.sql.functions as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession.builder.getOrCreate()\n",
    "df = spark.read.text(\"./book-asset.txt\")\n",
    "df = df.filter(F.col(\"value\") != \"\") # Remove empty rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(value='The Project Gutenberg eBook of Frankenstein, by Mary Wollstonecraft (Godwin) Shelley'),\n",
       " Row(value='This eBook is for the use of anyone anywhere in the United States and'),\n",
       " Row(value='most other parts of the world at no cost and with almost no restrictions'),\n",
       " Row(value='whatsoever. You may copy it, give it away or re-use it under the terms'),\n",
       " Row(value='of the Project Gutenberg License included with this eBook or online at')]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_counts = (\n",
    "    df.withColumn(\"word\", F.explode(F.split(F.col(\"value\"), \"\\s+\")))\n",
    "    .withColumn(\"word\", F.regexp_replace(\"word\", \"[^\\w]\", \"\"))\n",
    "    .groupBy(\"word\")\n",
    "    .count()\n",
    "    .sort(\"count\", ascending=False)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[Row(word='the', count=4073),\n",
       " Row(word='and', count=2998),\n",
       " Row(word='I', count=2840),\n",
       " Row(word='of', count=2748),\n",
       " Row(word='to', count=2154)]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_counts.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 25:==================================================>   (188 + 4) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+-----+\n",
      "|word|count|\n",
      "+----+-----+\n",
      "| the| 4073|\n",
      "| and| 2998|\n",
      "|   I| 2840|\n",
      "|  of| 2748|\n",
      "|  to| 2154|\n",
      "|  my| 1632|\n",
      "|   a| 1395|\n",
      "|  in| 1137|\n",
      "| was| 1020|\n",
      "|that| 1015|\n",
      "+----+-----+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "# Top 10\n",
    "word_counts.show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 27:=====================================================>(197 + 3) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+\n",
      "|count_all_words|\n",
      "+---------------+\n",
      "|          78214|\n",
      "+---------------+\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "# All words count\n",
    "word_counts.agg(F.sum(\"count\").alias(\"count_all_words\")).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 30:=============================================>        (170 + 4) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+\n",
      "|whale_count|\n",
      "+-----------+\n",
      "|          4|\n",
      "+-----------+\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "# Whale count\n",
    "word_counts.filter(F.col(\"word\").rlike(\"(?i)whale\")).agg(\n",
    "    F.sum(\"count\").alias(\"whale_count\")\n",
    ").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 33:==============================================>       (174 + 4) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique words:  8016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "# Unique count\n",
    "print(\"Unique words: \", word_counts.count())"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "ff4a9c5a471628d37aaf69a3c82b80059c9f280b7365140dc0897ed511569510"
  },
  "kernelspec": {
   "display_name": "Python 3.8.8 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}