MLwithPython_Classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "MLwithPython-Classification.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LNUM5vqFTpvm"
      },
      "source": [
        "## Classification"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aa0B4GryPjS4"
      },
      "source": [
        "### Binary Classification : Spam mail classification"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Xw_7nV3Ivox"
      },
      "source": [
        "import math\n",
        "import random\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XFJeyL5CQAPe"
      },
      "source": [
        "#### Download dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6NG9x-oARiCR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aa241c8b-67bf-46e2-843d-f6ed7c718075"
      },
      "source": [
        "!wget https://archive.ics.uci.edu/ml/machine-learning-databases/00228/smsspamcollection.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-01-07 05:41:45--  https://archive.ics.uci.edu/ml/machine-learning-databases/00228/smsspamcollection.zip\n",
            "Resolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\n",
            "Connecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 203415 (199K) [application/x-httpd-php]\n",
            "Saving to: ‘smsspamcollection.zip’\n",
            "\n",
            "smsspamcollection.z 100%[===================>] 198.65K   775KB/s    in 0.3s    \n",
            "\n",
            "2021-01-07 05:41:46 (775 KB/s) - ‘smsspamcollection.zip’ saved [203415/203415]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3DuFdpvfPyQH",
        "outputId": "5a00ec1f-2d0e-4553-c7e0-e3b9f3898212"
      },
      "source": [
        "!unzip smsspamcollection.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  smsspamcollection.zip\n",
            "  inflating: SMSSpamCollection       \n",
            "  inflating: readme                  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0ixXgU7fP1TU",
        "outputId": "d61e0fce-58e9-4410-86e6-219b333e21bf"
      },
      "source": [
        "!head SMSSpamCollection"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ham\tGo until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\n",
            "ham\tOk lar... Joking wif u oni...\n",
            "spam\tFree entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 87121 to receive entry question(std txt rate)T&C's apply 08452810075over18's\n",
            "ham\tU dun say so early hor... U c already then say...\n",
            "ham\tNah I don't think he goes to usf, he lives around here though\n",
            "spam\tFreeMsg Hey there darling it's been 3 week's now and no word back! I'd like some fun you up for it still? Tb ok! XxX std chgs to send, £1.50 to rcv\n",
            "ham\tEven my brother is not like to speak with me. They treat me like aids patent.\n",
            "ham\tAs per your request 'Melle Melle (Oru Minnaminunginte Nurungu Vettam)' has been set as your callertune for all Callers. Press *9 to copy your friends Callertune\n",
            "spam\tWINNER!! As a valued network customer you have been selected to receivea £900 prize reward! To claim call 09061701461. Claim code KL341. Valid 12 hours only.\n",
            "spam\tHad your mobile 11 months or more? U R entitled to Update to the latest colour mobiles with camera for Free! Call The Mobile Update Co FREE on 08002986030\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gk6VTlzjQDOO"
      },
      "source": [
        "#### Data preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TNQniRG3P8Gy"
      },
      "source": [
        "def tokenize(s):\n",
        "    tokens = s.split(' ')\n",
        "    return [t.rstrip('.') for t in tokens]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0cR91lGKRFsu"
      },
      "source": [
        "spam $\\to$ y=1\n",
        "\n",
        "ham  $\\to$ y=0"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MxuUnBexQK49"
      },
      "source": [
        "D = []\n",
        "\n",
        "with open('SMSSpamCollection') as fi:\n",
        "    for line in fi:\n",
        "        fields = line.strip('\\n').split('\\t')\n",
        "        x = tokenize(fields[1])\n",
        "        y = 1 if fields[0] == 'spam' else 0\n",
        "        D.append((x, y))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5fpvUbdYQL_u",
        "outputId": "e2a884cd-b54d-484f-e0e1-8d1c3da04b76"
      },
      "source": [
        "print(\"Number of data N=\",len(D))\n",
        "print(\"Example:\",D[10])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of data N= 5574\n",
            "Example: ([\"I'm\", 'gonna', 'be', 'home', 'soon', 'and', 'i', \"don't\", 'want', 'to', 'talk', 'about', 'this', 'stuff', 'anymore', 'tonight,', 'k?', \"I've\", 'cried', 'enough', 'today'], 0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CeP4vCDkRSvq"
      },
      "source": [
        "រៀបចំទិន្នន័យសម្រាប់កំណត់ប៉ារ៉ាម៉ែត្រម៉ូឌែល(training data) និងសម្រាប់វាយតម្លៃ(test data)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J3O6BVLoQQFS"
      },
      "source": [
        "import random\n",
        "random.shuffle(D)\n",
        "Dtrain = D[:5000]\n",
        "Dtest = D[5000:]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sU3nSLuGQXZ8"
      },
      "source": [
        "#### Logistic Regression Model with Python"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KhqPgFvxSU8E"
      },
      "source": [
        "កំណត់តម្លៃដំបូងនៃប៉ារ៉ាម៉ែត្ររបស់ម៉ូឌែលដោយ 0. គ្រប់កំប៉ូសង់របស់វា"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NrMVoqb6QUAQ"
      },
      "source": [
        "W = {}\n",
        "for x, y in Dtrain:\n",
        "    for a in x:\n",
        "        W.setdefault(a, 0.)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RQCkA3-DQaqD"
      },
      "source": [
        "def score(W, x):\n",
        "  score_ = 0.\n",
        "  for a in x:\n",
        "    score_ += W.get(a, 0.)\n",
        "  return score_"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "siHTIvOiQjph"
      },
      "source": [
        "def sigmoid(a):\n",
        "    if 0 <= a:\n",
        "        return 1 / (1 + math.exp(-a))\n",
        "    else:\n",
        "        return 1. - 1 / (1 + math.exp(a))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k_pDcq7vSgJf"
      },
      "source": [
        "កំណត់តម្លៃប៉ារ៉ាម៉ែត្រដោយប្រើ SGD "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O_bturIOQlzl"
      },
      "source": [
        "eta = 0.1\n",
        "for t in range(1000):\n",
        "    loss = 0.\n",
        "    for x, y in Dtrain:\n",
        "        pi = sigmoid(score(W, x))\n",
        "        for a in x:\n",
        "            W[a] += eta * (y - pi)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RVICtXcXSkdx"
      },
      "source": [
        "ប្រើ Accuracy ជារង្វាស់សម្រាប់វាយតម្លៃម៉ូឌែល"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dtZy0XiYQn3i"
      },
      "source": [
        "def accuracy(W, Dtest):\n",
        "    n = 0\n",
        "    for x, y in Dtest:\n",
        "      if score(W, x) > 0:\n",
        "        n += y\n",
        "      else:\n",
        "        n += 1-y\n",
        "    return n / len(Dtest)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_-bIEajJQrdR",
        "outputId": "8f828dc7-3852-4580-9373-8b358cc6a0f6"
      },
      "source": [
        "accuracy(W, Dtest)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9599303135888502"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 148
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vCmNxpXgSvzh"
      },
      "source": [
        "កំណត់ពាក្យដែលពិបាក និង ងាយស្រួល កំណត់សម្គាល់ spam mail តាមរយៈតម្លៃនៃប៉ារ៉ាម៉ែត្រ"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5JNlzPw8QtCm"
      },
      "source": [
        "F = sorted(W.items(), key=lambda x:x[1])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1L8Ola5VTBEr"
      },
      "source": [
        "ពាក្យដែលងាយមិនងាយសម្គាល់បានថាជា​ spam mail"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "de3_2dDgQvJn",
        "outputId": "f1e3d7ba-97a2-4c0e-b73a-289202eaba21"
      },
      "source": [
        "F[:20]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('Ok', -7.844640065364425),\n",
              " ('Okie', -6.552802501569811),\n",
              " ('Yup', -6.116795874425502),\n",
              " ('Lmao!nice', -5.30308385989646),\n",
              " ('me', -5.269310452138359),\n",
              " ('i', -5.110793621056064),\n",
              " ('later', -5.096197697390265),\n",
              " ('Thanx', -4.869065935087656),\n",
              " ('So', -4.857796290868639),\n",
              " (\"I'll\", -4.845496566658822),\n",
              " ('Anything', -4.7141710996292385),\n",
              " ('him', -4.657607060331255),\n",
              " ('Anytime', -4.602664106243611),\n",
              " ('Then', -4.590316544672357),\n",
              " ('lor', -4.586966172828678),\n",
              " ('ALRITE', -4.580703739976091),\n",
              " ('Havent', -4.5721573931901025),\n",
              " ('G.W.R', -4.570827983667447),\n",
              " ('645', -4.570827983667447),\n",
              " ('Beerage?', -4.570827983667447)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 150
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HaAwhW26TH5j"
      },
      "source": [
        "ពាក្យដែលងាយងាយស្រួលក្នុងការសម្គាល់បានថាជា​ spam mail"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tRrej-QEQwb1",
        "outputId": "3eab8218-b8fc-4348-dbec-135ea0699365"
      },
      "source": [
        "F[-20:]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('who', 4.234476188151321),\n",
              " ('mobile', 4.280981090729554),\n",
              " ('Call', 4.615510487246776),\n",
              " ('chat,', 4.616365531017088),\n",
              " ('sexy?', 4.6512154135683765),\n",
              " ('STOP', 4.834027267340355),\n",
              " ('look', 4.871824105338229),\n",
              " ('-', 5.120099283844329),\n",
              " ('darling', 5.142180755157025),\n",
              " ('FREE', 5.181877253173467),\n",
              " ('07090201529', 5.238470185798956),\n",
              " ('txt', 5.247757797403913),\n",
              " ('To', 5.364577714800223),\n",
              " ('Text', 5.6096358084199505),\n",
              " ('Reply', 5.787635787001819),\n",
              " ('&', 5.836134374631943),\n",
              " ('won', 5.939243177798654),\n",
              " ('Txt', 6.002198351200192),\n",
              " ('Luv', 6.453202186942911),\n",
              " (\"let's\", 6.781294267285314)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 151
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1U2-YfvSQymW"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9SFthTvneNwa"
      },
      "source": [
        "### Multi-class Classification : News Classification"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0k3-H1zZeYVh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ab39d5bc-ea3e-4a3a-d5fd-cef919b01c4e"
      },
      "source": [
        "from sklearn.datasets import fetch_20newsgroups\n",
        "categories = ['alt.atheism', 'talk.religion.misc','comp.graphics', 'sci.space']\n",
        "newsgroups_train = fetch_20newsgroups(subset='train', categories=categories)\n",
        "newsgroups_test = fetch_20newsgroups(subset='test', categories=categories)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading 20news dataset. This may take a few minutes.\n",
            "Downloading dataset from https://ndownloader.figshare.com/files/5975967 (14 MB)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y1OC9stqitkj",
        "outputId": "11ddeef9-0554-4060-f2ce-224782e89c87"
      },
      "source": [
        "newsgroups_train['data'][0],newsgroups_test['data'][0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(\"From: rych@festival.ed.ac.uk (R Hawkes)\\nSubject: 3DS: Where did all the texture rules go?\\nLines: 21\\n\\nHi,\\n\\nI've noticed that if you only save a model (with all your mapping planes\\npositioned carefully) to a .3DS file that when you reload it after restarting\\n3DS, they are given a default position and orientation.  But if you save\\nto a .PRJ file their positions/orientation are preserved.  Does anyone\\nknow why this information is not stored in the .3DS file?  Nothing is\\nexplicitly said in the manual about saving texture rules in the .PRJ file. \\nI'd like to be able to read the texture rule information, does anyone have \\nthe format for the .PRJ file?\\n\\nIs the .CEL file format available from somewhere?\\n\\nRych\\n\\n======================================================================\\nRycharde Hawkes\\t\\t\\t\\temail: rych@festival.ed.ac.uk\\nVirtual Environment Laboratory\\nDept. of Psychology\\t\\t\\tTel  : +44 31 650 3426\\nUniv. of Edinburgh\\t\\t\\tFax  : +44 31 667 0150\\n======================================================================\\n\",\n",
              " 'From: prb@access.digex.net (Pat)\\nSubject: Re: Near Miss Asteroids (Q)\\nOrganization: Express Access Online Communications, Greenbelt, MD USA\\nLines: 4\\nDistribution: sci\\nNNTP-Posting-Host: access.digex.net\\n\\n\\nTRry the SKywatch project in  Arizona.\\n\\npat\\n')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 153
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WDvnyGiwiczB"
      },
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MVey6HPLi_Tv",
        "outputId": "80e73940-733e-43f8-f740-5104acd2c006"
      },
      "source": [
        "vectorizer = CountVectorizer()\n",
        "Xtrain = vectorizer.fit_transform(newsgroups_train.data)\n",
        "Ytrain = newsgroups_train.target\n",
        "Ytrain_onehot = np.eye(len(categories))[Ytrain]\n",
        "\n",
        "\n",
        "Xtest = vectorizer.transform(newsgroups_test.data)\n",
        "Ytest = newsgroups_test.target\n",
        "Ytest_onehot = np.eye(len(categories))[Ytest]\n",
        "\n",
        "print(\"Training Data\")\n",
        "print(\"Number of Training Data (N) = %d\"%(Xtrain.shape[0]))\n",
        "print(\"Feature vector \",(Xtrain.shape))\n",
        "print(\"Train Label:\",(Ytrain_onehot.shape))\n",
        "\n",
        "print(\"Test Data\")\n",
        "print(\"Number of Test Data (N) = %d\"%(Xtest.shape[0]))\n",
        "print(\"Feature vector \",(Xtest.shape))\n",
        "print(\"Test Label:\",(Ytest_onehot.shape))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training Data\n",
            "Number of Training Data (N) = 2034\n",
            "Feature vector  (2034, 34118)\n",
            "Train Label: (2034, 4)\n",
            "Test Data\n",
            "Number of Test Data (N) = 1353\n",
            "Feature vector  (1353, 34118)\n",
            "Test Label: (1353, 4)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qy7dH-IqjGk2",
        "outputId": "5a3c7465-3025-4157-adcd-7c55ef875de0"
      },
      "source": [
        "from sklearn.linear_model import SGDClassifier\n",
        "\n",
        "model = SGDClassifier(loss='log')\n",
        "model.fit(Xtrain, Ytrain)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SGDClassifier(alpha=0.0001, average=False, class_weight=None,\n",
              "              early_stopping=False, epsilon=0.1, eta0=0.0, fit_intercept=True,\n",
              "              l1_ratio=0.15, learning_rate='optimal', loss='log', max_iter=1000,\n",
              "              n_iter_no_change=5, n_jobs=None, penalty='l2', power_t=0.5,\n",
              "              random_state=None, shuffle=True, tol=0.001,\n",
              "              validation_fraction=0.1, verbose=0, warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 156
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mu2mAeKUjH7g",
        "outputId": "2e01c1dd-b930-4bac-da5d-8f7fef6a658e"
      },
      "source": [
        "model.score(Xtest,Ytest)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8529194382852919"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 157
        }
      ]
    }
  ]
}