Demo20181020.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bag of Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['How to format my hard disk', ' Hard disk format problems ']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "content = [\"How to format my hard disk\", \" Hard disk format problems \"]\n",
    "content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "?CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = CountVectorizer(min_df = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = vectorizer.fit_transform(content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<2x7 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 10 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['disk', 'format', 'hard', 'how', 'my', 'problems', 'to']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1, 1, 1, 1, 1, 0, 1],\n",
       "       [1, 1, 1, 0, 0, 1, 0]], dtype=int64)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(vectorizer.get_feature_names())\n",
    "X.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 計算查詢關鍵字對文章的距離"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "01.txt  02.txt  03.txt  04.txt  05.txt\r\n"
     ]
    }
   ],
   "source": [
    "%ls data/toy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['data/toy/01.txt',\n",
       " 'data/toy/02.txt',\n",
       " 'data/toy/03.txt',\n",
       " 'data/toy/04.txt',\n",
       " 'data/toy/05.txt']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "path  = 'data/toy'\n",
    "[os.path.join(path, f) for f in os.listdir(path)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "posts = [open(os.path.join(path, f)).read() for f in os.listdir(path)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer(min_df = 1)\n",
    "X_train = vectorizer.fit_transform(posts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<5x25 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 33 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['about', 'actually', 'capabilities', 'contains', 'data', 'databases', 'images', 'imaging', 'interesting', 'is', 'it', 'learning', 'machine', 'most', 'much', 'not', 'permanently', 'post', 'provide', 'safe', 'storage', 'store', 'stuff', 'this', 'toy']\n"
     ]
    }
   ],
   "source": [
    "# Tell you how many unique words\n",
    "print(vectorizer.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0,\n",
       "        1, 1, 1],\n",
       "       [0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0,\n",
       "        0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0,\n",
       "        0, 0, 0],\n",
       "       [0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,\n",
       "        0, 0, 0],\n",
       "       [0, 0, 0, 0, 3, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3,\n",
       "        0, 0, 0]], dtype=int64)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 25)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#samples: 5, #features: 25\n",
      "['about', 'actually', 'capabilities', 'contains', 'data', 'databases', 'images', 'imaging', 'interesting', 'is', 'it', 'learning', 'machine', 'most', 'much', 'not', 'permanently', 'post', 'provide', 'safe', 'storage', 'store', 'stuff', 'this', 'toy']\n"
     ]
    }
   ],
   "source": [
    "num_samples, num_features = X_train.shape\n",
    "\n",
    "print(\"#samples: %d, #features: %d\" % (num_samples,num_features)) \n",
    "print(vectorizer.get_feature_names())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_post = 'imaging database'\n",
    "new_post_vec = vectorizer.transform([new_post])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x25 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 1 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_post_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['about', 'actually', 'capabilities', 'contains', 'data', 'databases', 'images', 'imaging', 'interesting', 'is', 'it', 'learning', 'machine', 'most', 'much', 'not', 'permanently', 'post', 'provide', 'safe', 'storage', 'store', 'stuff', 'this', 'toy']\n"
     ]
    }
   ],
   "source": [
    "# Tell you how many unique words\n",
    "print(vectorizer.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_post_vec.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 計算查詢關鍵字對文章的距離 (距離一：歐式距離)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy as sp\n",
    "\n",
    "def dist_raw(v1, v2):\n",
    "    delta = v1-v2\n",
    "    return sp.linalg.norm(delta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.4142135623730951"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy \n",
    "a = numpy.array([0,0,1,1,0])\n",
    "b = numpy.array([1,0,1,0,0])\n",
    "dist_raw(a,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.4142135623730951"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import math\n",
    "a = [0,0,1,1,0]\n",
    "b = [1,0,1,0,0]\n",
    "d = []\n",
    "for i in range(0,len(a)):\n",
    "    d.append(a[i] - b[i])\n",
    "math.sqrt(sum([ele ** 2for ele in d]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.4142135623730951"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = numpy.array([0,0,1,1,0])\n",
    "b = numpy.array([1,0,1,0,0])\n",
    "math.sqrt(sum((a - b)** 2) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy as sp\n",
    "\n",
    "def dist_raw(v1, v2):\n",
    "    delta = v1-v2\n",
    "    return sp.linalg.norm(delta.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "best_doc = None\n",
    "best_dist = 999\n",
    "best_i = None\n",
    "num_samples = len(posts)\n",
    "\n",
    "for i in range(0, num_samples):\n",
    "    post = posts[i]\n",
    "    if post==new_post:\n",
    "        continue\n",
    "    post_vec = X_train.getrow(i)\n",
    "    d = dist(post_vec, new_post_vec)\n",
    "    print(\"=== Post %i with dist=%.2f: %s\"%(i, d, post))\n",
    "    if d<best_dist:\n",
    "        best_dist = d\n",
    "        best_i = i\n",
    "print(\"Best post is %i with dist=%.2f\"%(best_i, best_dist))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 This is a toy post about machine learning. Actually, it contains not much interesting stuff. 3.872983346207417\n",
      "1 Imaging databases provide storage capabilities. 2.0\n",
      "2 Most imaging databases safe images permanently. 2.23606797749979\n",
      "3 Imaging databases store data. 1.7320508075688772\n",
      "4 Imaging databases store data. Imaging databases store data. Imaging databases store data. 5.5677643628300215\n"
     ]
    }
   ],
   "source": [
    "for i in range(0,5):\n",
    "    print(i, posts[i], dist_raw(X_train[i], new_post_vec))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 計算查詢關鍵字對文章的距離 (距離二：標準化後的歐式距離)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dist(v1, v2):\n",
    "    v1_normalized  = v1 / sp.linalg.norm(v1.toarray()) \n",
    "    v2_normalized  = v2 / sp.linalg.norm(v2.toarray())\n",
    "    delta = v1_normalized - v2_normalized\n",
    "    return sp.linalg.norm(delta.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.57735027, 0.57735027, 0.57735027, 0.        , 0.        ])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = numpy.array([1,1,1,0,0])\n",
    "#math.sqrt(3)\n",
    "a / sp.linalg.norm(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.57735027, 0.57735027, 0.57735027, 0.        , 0.        ])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = numpy.array([3,3,3,0,0])\n",
    "b / sp.linalg.norm(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 This is a toy post about machine learning. Actually, it contains not much interesting stuff. 1.4142135623730951\n",
      "1 Imaging databases provide storage capabilities. 1.0514622242382672\n",
      "2 Most imaging databases safe images permanently. 1.0878894332937856\n",
      "3 Imaging databases store data. 1.0\n",
      "4 Imaging databases store data. Imaging databases store data. Imaging databases store data. 1.0\n"
     ]
    }
   ],
   "source": [
    "for i in range(0,5):\n",
    "    print(i, posts[i], dist(X_train[i], new_post_vec))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['about', 'actually', 'capabilities', 'contains', 'data', 'databases', 'images', 'imaging', 'interesting', 'is', 'it', 'learning', 'machine', 'most', 'much', 'not', 'permanently', 'post', 'provide', 'safe', 'storage', 'store', 'stuff', 'this', 'toy']\n"
     ]
    }
   ],
   "source": [
    "# Tell you how many unique words\n",
    "print(vectorizer.get_feature_names())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 移除停用詞"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nltk in /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages\n",
      "Requirement already satisfied: six in /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages (from nltk)\n",
      "\u001b[33mYou are using pip version 9.0.1, however version 18.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "! pip install nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['actually', 'capabilities', 'contains', 'data', 'databases', 'images', 'imaging', 'interesting', 'learning', 'machine', 'permanently', 'post', 'provide', 'safe', 'storage', 'store', 'stuff', 'toy']\n",
      "(5, 18)\n"
     ]
    }
   ],
   "source": [
    "vectorizer = CountVectorizer(min_df=1, stop_words='english')\n",
    "X = vectorizer.fit_transform(posts)\n",
    "print(vectorizer.get_feature_names())\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_post = 'imaging database'\n",
    "new_post_vec = vectorizer.transform([new_post])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_post_vec.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 This is a toy post about machine learning. Actually, it contains not much interesting stuff. 1.4142135623730951\n",
      "1 Imaging databases provide storage capabilities. 1.0514622242382672\n",
      "2 Most imaging databases safe images permanently. 1.0514622242382672\n",
      "3 Imaging databases store data. 1.0\n",
      "4 Imaging databases store data. Imaging databases store data. Imaging databases store data. 1.0\n"
     ]
    }
   ],
   "source": [
    "for i in range(0,5):\n",
    "    print(i, posts[i], dist(X[i], new_post_vec))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk.stem\n",
    "s = nltk.stem.SnowballStemmer('english')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'graphic'"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.stem('graphics')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "imag\n",
      "imag\n",
      "imagin\n",
      "imagin\n"
     ]
    }
   ],
   "source": [
    "print(s.stem(\"imaging\"))\n",
    "print(s.stem(\"image\"))\n",
    "\n",
    "print(s.stem(\"imagination\"))\n",
    "print(s.stem(\"imagine\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk.stem\n",
    "english_stemmer = nltk.stem.SnowballStemmer('english')\n",
    "\n",
    "class StemmedCountVectorizer(CountVectorizer):\n",
    "    def build_analyzer(self):\n",
    "        analyzer = super(StemmedCountVectorizer, self).build_analyzer()\n",
    "        return lambda doc: (english_stemmer.stem(w) for w in analyzer(doc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['actual', 'capabl', 'contain', 'data', 'databas', 'imag', 'interest', 'learn', 'machin', 'perman', 'post', 'provid', 'safe', 'storag', 'store', 'stuff', 'toy']\n",
      "(5, 17)\n"
     ]
    }
   ],
   "source": [
    "vectorizer = StemmedCountVectorizer(min_df=1, stop_words='english')\n",
    "X = vectorizer.fit_transform(posts)\n",
    "print(vectorizer.get_feature_names())\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_post = 'imaging database'\n",
    "new_post_vec = vectorizer.transform([new_post])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_post_vec.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 This is a toy post about machine learning. Actually, it contains not much interesting stuff. 1.414213562373095\n",
      "1 Imaging databases provide storage capabilities. 0.8573732768944039\n",
      "2 Most imaging databases safe images permanently. 0.6296288974669553\n",
      "3 Imaging databases store data. 0.7653668647301795\n",
      "4 Imaging databases store data. Imaging databases store data. Imaging databases store data. 0.7653668647301795\n"
     ]
    }
   ],
   "source": [
    "for i in range(0,5):\n",
    "    print(i, posts[i], dist(X[i], new_post_vec))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TF-IDF  詞頻矩陣"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['actually', 'capabilities', 'contains', 'data', 'databases', 'images', 'imaging', 'interesting', 'learning', 'machine', 'permanently', 'post', 'provide', 'safe', 'storage', 'store', 'stuff', 'toy']\n",
      "(5, 18)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "vectorizer = TfidfVectorizer(min_df=1, stop_words='english')\n",
    "X = vectorizer.fit_transform(posts)\n",
    "print(vectorizer.get_feature_names())\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_post = 'imaging database'\n",
    "new_post_vec = vectorizer.transform([new_post])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 This is a toy post about machine learning. Actually, it contains not much interesting stuff. 1.4142135623730951\n",
      "1 Imaging databases provide storage capabilities. 1.187009812033225\n",
      "2 Most imaging databases safe images permanently. 1.187009812033225\n",
      "3 Imaging databases store data. 1.091020922163783\n",
      "4 Imaging databases store data. Imaging databases store data. Imaging databases store data. 1.091020922163783\n"
     ]
    }
   ],
   "source": [
    "for i in range(0,5):\n",
    "    print(i, posts[i], dist(X[i], new_post_vec))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "class StemmedTfidfVectorizer(TfidfVectorizer):\n",
    "    def build_analyzer(self):\n",
    "        analyzer = super(TfidfVectorizer,self).build_analyzer()\n",
    "        return lambda doc: (english_stemmer.stem(w) for w in analyzer(doc))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['actual', 'capabl', 'contain', 'data', 'databas', 'imag', 'interest', 'learn', 'machin', 'perman', 'post', 'provid', 'safe', 'storag', 'store', 'stuff', 'toy']\n",
      "(5, 17)\n"
     ]
    }
   ],
   "source": [
    "vectorizer = StemmedTfidfVectorizer(min_df=1, stop_words='english')\n",
    "X = vectorizer.fit_transform(posts)\n",
    "print(vectorizer.get_feature_names())\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_post = 'imaging database'\n",
    "new_post_vec = vectorizer.transform([new_post])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 This is a toy post about machine learning. Actually, it contains not much interesting stuff. 1.4142135623730951\n",
      "1 Imaging databases provide storage capabilities. 1.0789758507558254\n",
      "2 Most imaging databases safe images permanently. 0.859044512133176\n",
      "3 Imaging databases store data. 0.924634506718001\n",
      "4 Imaging databases store data. Imaging databases store data. Imaging databases store data. 0.924634506718001\n"
     ]
    }
   ],
   "source": [
    "for i in range(0,5):\n",
    "    print(i, posts[i], dist(X[i], new_post_vec))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 中文詞頻矩陣"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Uninstalling jieba-0.39:\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba-0.39.dist-info/INSTALLER\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba-0.39.dist-info/METADATA\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba-0.39.dist-info/RECORD\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba-0.39.dist-info/WHEEL\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba-0.39.dist-info/top_level.txt\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/__init__.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/__init__.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/__main__.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/__pycache__/__init__.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/__pycache__/__main__.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/__pycache__/_compat.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/_compat.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/analyse/__init__.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/analyse/__init__.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/analyse/__pycache__/__init__.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/analyse/__pycache__/analyzer.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/analyse/__pycache__/textrank.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/analyse/__pycache__/tfidf.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/analyse/analyzer.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/analyse/analyzer.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/analyse/idf.txt\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/analyse/textrank.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/analyse/textrank.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/analyse/tfidf.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/dict.txt\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/finalseg/__init__.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/finalseg/__init__.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/finalseg/__pycache__/__init__.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/finalseg/__pycache__/prob_emit.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/finalseg/__pycache__/prob_start.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/finalseg/__pycache__/prob_trans.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/finalseg/prob_emit.p\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/finalseg/prob_emit.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/finalseg/prob_emit.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/finalseg/prob_start.p\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/finalseg/prob_start.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/finalseg/prob_start.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/finalseg/prob_trans.p\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/finalseg/prob_trans.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/finalseg/prob_trans.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/__init__.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/__init__.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/__pycache__/__init__.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/__pycache__/char_state_tab.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/__pycache__/prob_emit.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/__pycache__/prob_start.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/__pycache__/prob_trans.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/__pycache__/viterbi.cpython-36.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/char_state_tab.p\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/char_state_tab.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/char_state_tab.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/prob_emit.p\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/prob_emit.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/prob_emit.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/prob_start.p\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/prob_start.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/prob_start.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/prob_trans.p\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/prob_trans.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/prob_trans.pyc\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/viterbi.py\n",
      "  /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/jieba/posseg/viterbi.pyc\n",
      "Proceed (y/n)? ^C\n",
      "\u001b[31mOperation cancelled by user\u001b[0m\n",
      "\u001b[33mYou are using pip version 9.0.1, however version 18.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "! pip uninstall jieba"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 繁體中文版jieba\n",
    "- https://github.com/ldkrsi/jieba-zh_TW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "柯文哲\n",
      "為\n",
      "了\n",
      "大巨蛋\n",
      "一事\n",
      "找\n",
      "趙藤雄\n",
      "算帳\n"
     ]
    }
   ],
   "source": [
    "import jieba\n",
    "jieba.load_userdict('userdict.txt')\n",
    "\n",
    "for w in jieba.cut('柯文哲為了大巨蛋一事找趙藤雄算帳'):\n",
    "    print(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['柯文哲 為 了 大巨蛋 一事 找 趙藤雄 算帳', '柯P 將不在 大巨蛋 舉辦 世運會']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = ['柯文哲為了大巨蛋一事找趙藤雄算帳', '柯P將不在大巨蛋舉辦世運會']\n",
    "\n",
    "courpus  = []\n",
    "for s in a:\n",
    "    courpus.append(' '.join(jieba.cut(s)))\n",
    "courpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['柯文哲 為 了 大巨蛋 一事 找 趙藤雄 算帳', '柯P 將不在 大巨蛋 舉辦 世運會']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus = [' '.join(jieba.cut(s)) for s in a]\n",
    "corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer(min_df = 1)\n",
    "X = vectorizer.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<2x9 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 10 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['一事', '世運會', '大巨蛋', '將不在', '柯p', '柯文哲', '算帳', '舉辦', '趙藤雄']\n"
     ]
    }
   ],
   "source": [
    "print(vectorizer.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1 0 1 0 0 1 1 0 1]\n",
      " [0 1 1 1 1 0 0 1 0]]\n"
     ]
    }
   ],
   "source": [
    "print(X.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy as sp\n",
    "def dist(v1, v2):\n",
    "    v1_normalized  = v1 / sp.linalg.norm(v1.toarray()) \n",
    "    v2_normalized  = v2 / sp.linalg.norm(v2.toarray())\n",
    "    delta = v1_normalized - v2_normalized\n",
    "    return sp.linalg.norm(delta.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.2649110640673518"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dist(X[0],X[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "keyword = '郭雪芙'\n",
    "res = requests.get('https://zh.wikipedia.org/wiki/{}'.format(keyword))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "soup = BeautifulSoup(res.text, 'lxml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['郭雪芙', 'Puff Kuo']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "synonym = []\n",
    "for p in soup.select('.mw-parser-output p'):\n",
    "    for b in p.select('b'):\n",
    "        synonym.append(b.text)\n",
    "synonym"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## synonyms.txt\n",
    "- 柯文哲/柯P/柯p\n",
    "- 郭雪芙/puff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'puff': '郭雪芙', '柯p': '柯文哲'}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "synonym_dic = {}\n",
    "for s in open('synonyms.txt'):\n",
    "    synonym = s.strip().split('/')\n",
    "    for w in synonym[1:]:\n",
    "        synonym_dic[w.lower()]  = synonym[0]\n",
    "synonym_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['一事', '世運會', '大巨蛋', '將不在', '柯文哲', '算帳', '舉辦', '趙藤雄']\n"
     ]
    }
   ],
   "source": [
    "import nltk.stem\n",
    "\n",
    "class SynonymCountVectorizer(CountVectorizer):\n",
    "    def build_analyzer(self):\n",
    "        analyzer = super(SynonymCountVectorizer, self).build_analyzer()\n",
    "        return lambda doc: (synonym_dic.get(w, w) for w in analyzer(doc))\n",
    "\n",
    "vectorizer = SynonymCountVectorizer()\n",
    "X = vectorizer.fit_transform(corpus)\n",
    "print(vectorizer.get_feature_names())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 1, 0, 1, 1, 0, 1],\n",
       "       [0, 1, 1, 1, 1, 0, 1, 0]], dtype=int64)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['世運會', '大巨蛋', '柯文哲', '算帳', '舉辦', '趙藤雄']\n"
     ]
    }
   ],
   "source": [
    "stopwords = ['為了', '一事', '將不在']\n",
    "\n",
    "vectorizer = SynonymCountVectorizer(stop_words=stopwords)\n",
    "X = vectorizer.fit_transform(corpus)\n",
    "print(vectorizer.get_feature_names())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "content = [ '2天才女童到澳洲深造 竟全身傷疑受虐',\n",
    "'來自台灣、現在只有9歲與7歲的2名兒童，上周六(13日)流落於澳洲街頭時被當地警方發現帶回安置，懷疑受虐和人口販運有關，隨即通知我國警方，不過經調查發現，這2名高智商的女童，其實是由父親委託友人帶去澳洲自主學習，父親表示並非遭到人口販運，小姐妹數理方面優異，當初委託朋友帶到澳洲深造。']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = [' '.join(jieba.cut(s)) for s in content]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = CountVectorizer()\n",
    "X = vectorizer.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy as sp\n",
    "\n",
    "def dist_raw(v1, v2):\n",
    "    delta = v1-v2\n",
    "    return sp.linalg.norm(delta.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7.681145747868608"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dist_raw(X[0], X[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0],\n",
       "       [1, 1, 1, 2, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "        1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 2, 2, 1,\n",
       "        1, 1, 1]], dtype=int64)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "content=  ['我喜歡看電視不喜歡看電影','我不喜歡看電視也不喜歡看電影']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = [' '.join(jieba.cut(s)) for s in content]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['我 喜歡 看 電視 不 喜歡 看 電影', '我 不 喜歡 看 電視 也 不 喜歡 看 電影']"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "content = ['柯文哲為了大巨蛋一事找趙藤雄算帳', '柯P將不在大巨蛋舉辦世運會']\n",
    "corpus = [' '.join(jieba.cut(s)) for s in content]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['一事', '世運會', '大巨蛋', '將不在', '柯文哲', '算帳', '舉辦', '趙藤雄']\n"
     ]
    }
   ],
   "source": [
    "import nltk.stem\n",
    "\n",
    "class SynonymCountVectorizer(CountVectorizer):\n",
    "    def build_analyzer(self):\n",
    "        analyzer = super(SynonymCountVectorizer, self).build_analyzer()\n",
    "        return lambda doc: (synonym_dic.get(w, w) for w in analyzer(doc))\n",
    "\n",
    "vectorizer = SynonymCountVectorizer()\n",
    "X = vectorizer.fit_transform(corpus)\n",
    "print(vectorizer.get_feature_names())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 1, 0, 1, 1, 0, 1],\n",
       "       [0, 1, 1, 1, 1, 0, 1, 0]], dtype=int64)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1. , 0.4],\n",
       "       [0.4, 1. ]])"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "cs = cosine_similarity(X)\n",
    "cs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0. , 0.6],\n",
       "       [0.6, 0. ]])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics.pairwise import cosine_distances\n",
    "cs = cosine_distances(X)\n",
    "cs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 新聞推薦系統"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "news = pandas.read_excel('https://raw.githubusercontent.com/ywchiu/pytextmining/master/data/20171214news.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba\n",
    "corpus = []\n",
    "titles = []\n",
    "for rec in news.iterrows():\n",
    "    #print(rec)\n",
    "    #break\n",
    "    corpus.append(' '.join(jieba.cut(rec[1].content)))\n",
    "    titles.append(rec[1].title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'新增 ： 立委 說 法民進 黨 立法院 黨團 預計 在 明天 的 院會 中 ， 讓 改制 農田 水利 會 的 《 農田 水利 會 組織 通則 》 修正 草案 付委 ， 國民黨 主席 吳敦義 昨在 中常 會上 表示 ， 對 於 蔡 政府 這種 巧取 豪奪 水利 會資產 作法 要 全力 反對 ， 「 甚至 要 有 夜宿 立法院 的 準備 」 。 對此 ， 民進 黨 立委 蔡易餘 今天 早上 發臉 書 表示 ， 吳敦義的 軍令狀 已 經破功 了 ， 「 因為 我們 已 經先 佔 議場 前 ， 24 小時 固守 活動 開始 ！ ！ 」 民進 黨 立院 黨團書 記長 何欣純 今早 表示 ， 目前 排班 兩小時 一班 ， 夜晚 則由 男性 立委 負責 駐守 ， 不換班 。 民進 黨團 總召 柯建銘 今早 也 特別 叮嚀 綠委 ， 要 去 洗手 間 時 ， 必須 要 確認 一定 要 有 一個 人 駐守 在 議場 門口 ， 不能 全部 離開 。 今早 也 有 綠委說 ， 好險 今天 天氣 回暖 ， 坐在 這裡 沒有 問題 。 （ 黃 信維 ／ 台北 報導 ） 發稿 時間 ： 09 : 09 更新 時間 ： 09 : 23 \\xa0 想 知道 更 多 ， 一定 要 看 … … 國 防部 解約慶富 要 提告 \\u3000 馮世 寬 怒 嗆 三遍 ： 請 便 罷免 案將 投票 \\u3000 李遠 哲今 再度 現身力 挺 黃 國昌 【 歷史 上 的 今天 】 賴香伶當 選北市 勞動 局長 \\u3000 周錫瑋花 百萬蓋 縣長 辦 公室'"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "899"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer(min_df = 1)\n",
    "X = vectorizer.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<899x38653 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 128155 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_distances\n",
    "cs = cosine_distances(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(899, 899)"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.96829548, 0.94683527],\n",
       "       [0.96829548, 0.        , 0.9739099 ],\n",
       "       [0.94683527, 0.9739099 , 0.        ]])"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cs[0:3,0:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 3, 1, 0, 4, 5])"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "a = np.array([80,70,56,63,89,90])\n",
    "a.argsort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_pos = cs[  0  ,  : ].argsort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'【更新】水利會改官派明闖關\\u3000綠委24小時前顧議場大門防藍突襲'"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titles[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "反對水利會改制　吳敦義下令：藍委做好夜宿立院抗爭準備 0.6321547504275353\n",
      "農田水利會改公務機關　蔡英文：這不是綁樁 0.6724128474406619\n",
      "罷免案將投票　李遠哲今再度現身力挺黃國昌 0.7771295373006972\n",
      "在野黨突襲表決《勞基法》修正案無效　民進黨烙人成功擋下 0.778385915154946\n",
      "【搏感情動畫】台電每年30億敦親睦鄰費　協助立委選民服務 0.7844869521946108\n",
      "國防部解約慶富要提告　馮世寬怒嗆三遍：請便 0.7946408559172016\n"
     ]
    }
   ],
   "source": [
    "for idx in sorted_pos[1:11]:\n",
    "    if cs[0,idx] < 0.8:\n",
    "        print(titles[idx], cs[0,idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getSimiliarArticle(pos):\n",
    "    print('查詢文章:',titles[pos])\n",
    "    sorted_pos = cs[  pos  ,  : ].argsort()\n",
    "    for idx in sorted_pos[1:11]:\n",
    "        if cs[pos,idx] < 0.8:\n",
    "            print('關聯文章:', titles[idx], cs[pos,idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "查詢文章: 【更新】水利會改官派明闖關　綠委24小時前顧議場大門防藍突襲\n",
      "關聯文章: 反對水利會改制　吳敦義下令：藍委做好夜宿立院抗爭準備 0.6321547504275353\n",
      "關聯文章: 農田水利會改公務機關　蔡英文：這不是綁樁 0.6724128474406619\n",
      "關聯文章: 罷免案將投票　李遠哲今再度現身力挺黃國昌 0.7771295373006972\n",
      "關聯文章: 在野黨突襲表決《勞基法》修正案無效　民進黨烙人成功擋下 0.778385915154946\n",
      "關聯文章: 【搏感情動畫】台電每年30億敦親睦鄰費　協助立委選民服務 0.7844869521946108\n",
      "關聯文章: 國防部解約慶富要提告　馮世寬怒嗆三遍：請便 0.7946408559172016\n"
     ]
    }
   ],
   "source": [
    "getSimiliarArticle(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "查詢文章: 黃國昌：長期目標要消滅國民黨　不支持柯文哲的兩岸一家親\n",
      "關聯文章: 「兩蔣時代」超譯　網友：不准傷皇城內的和氣！ 0.5791275247889435\n",
      "關聯文章: 罷免案將投票　李遠哲今再度現身力挺黃國昌 0.595349265179326\n",
      "關聯文章: 黃國昌若被罷免會更強　他：成為選輸北市的阿扁 0.6343677782259318\n",
      "關聯文章: 罷昌案周六投票　時代力量全力澄清不實謠言 0.6561575542619899\n",
      "關聯文章: 【聲援片】沈發惠呼籲罷昌案投「不同意」　黃國昌：謝謝您 0.7538388282723372\n"
     ]
    }
   ],
   "source": [
    "getSimiliarArticle(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "查詢文章: 桃園工廠火勢熄滅　員工宿舍內發現一堆白骨\n",
      "關聯文章: 【不斷更新】桃園工廠惡火撲滅 6人仍失聯宿舍內發現一堆白骨 0.36731278412423196\n",
      "關聯文章: 汽車用品大廠「矽卡」燒毀　資本額達2億 0.5443494259494621\n",
      "關聯文章: 桃園工廠大火6員工失聯　家屬焦急等待 0.6699125279936771\n",
      "關聯文章: 醉女盧小小　女警帥爆！突然霸氣送她側摔　 0.7894412780969211\n"
     ]
    }
   ],
   "source": [
    "getSimiliarArticle(25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 文章分群"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "news = pandas.read_excel('https://raw.githubusercontent.com/ywchiu/pytextmining/master/data/20150628news.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>description</th>\n",
       "      <th>pubdate</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>社會/生活</td>\n",
       "      <td>新北市八仙水上樂園昨晚發生粉塵爆炸，新北市衛生局統計到目前為止，由救護車送醫再加上自行送醫的...</td>\n",
       "      <td>Sun, 28 Jun 2015 07:40:00 +0800</td>\n",
       "      <td>八仙塵爆  五相關人依公共危險重傷害法辦</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>社會/生活</td>\n",
       "      <td>新北市八仙樂園昨天晚上(6/27)舉辦活動，過程中噴灑大量玉米粉而引發粉塵爆炸，根據最新統計...</td>\n",
       "      <td>Sun, 28 Jun 2015 07:40:00 +0800</td>\n",
       "      <td>八仙樂園意外 病患持續增加中</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>財經/要聞</td>\n",
       "      <td>希臘債務違約限期逼近，資金持續外流。路透社引述三間銀行的消息指出，希臘國內有3分之1的自動櫃...</td>\n",
       "      <td>Sun, 28 Jun 2015 07:40:00 +0800</td>\n",
       "      <td>希臘國內三分一自動櫃員機現金短缺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>社會/生活</td>\n",
       "      <td>台鐵「新基隆車站」第一階段工程本月底完成，台鐵表示，明天（29號）啟用後，由於南站周邊道路尚...</td>\n",
       "      <td>Sun, 28 Jun 2015 07:40:00 +0800</td>\n",
       "      <td>台鐵新基隆車站29日正式啟用</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>財經/要聞</td>\n",
       "      <td>《中國時報》\\n●樂園變煉獄 派對驚爆 逾300人遭火紋身\\n八仙樂園昨晚舉辦「COLOR ...</td>\n",
       "      <td>Sun, 28 Jun 2015 07:38:17 +0800</td>\n",
       "      <td>6月28日各報頭版要聞</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  category                                        description  \\\n",
       "0    社會/生活  新北市八仙水上樂園昨晚發生粉塵爆炸，新北市衛生局統計到目前為止，由救護車送醫再加上自行送醫的...   \n",
       "1    社會/生活  新北市八仙樂園昨天晚上(6/27)舉辦活動，過程中噴灑大量玉米粉而引發粉塵爆炸，根據最新統計...   \n",
       "2    財經/要聞  希臘債務違約限期逼近，資金持續外流。路透社引述三間銀行的消息指出，希臘國內有3分之1的自動櫃...   \n",
       "3    社會/生活  台鐵「新基隆車站」第一階段工程本月底完成，台鐵表示，明天（29號）啟用後，由於南站周邊道路尚...   \n",
       "4    財經/要聞  《中國時報》\\n●樂園變煉獄 派對驚爆 逾300人遭火紋身\\n八仙樂園昨晚舉辦「COLOR ...   \n",
       "\n",
       "                           pubdate                 title  \n",
       "0  Sun, 28 Jun 2015 07:40:00 +0800  八仙塵爆  五相關人依公共危險重傷害法辦  \n",
       "1  Sun, 28 Jun 2015 07:40:00 +0800        八仙樂園意外 病患持續增加中  \n",
       "2  Sun, 28 Jun 2015 07:40:00 +0800      希臘國內三分一自動櫃員機現金短缺  \n",
       "3  Sun, 28 Jun 2015 07:40:00 +0800        台鐵新基隆車站29日正式啟用  \n",
       "4  Sun, 28 Jun 2015 07:38:17 +0800           6月28日各報頭版要聞  "
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = []\n",
    "titles = []\n",
    "for rec in news.iterrows():\n",
    "    corpus.append(' '.join(jieba.cut(rec[1].description)))\n",
    "    titles.append(rec[1].title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer()\n",
    "X = vectorizer.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<147x12825 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 23783 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_distances\n",
    "n_cosine_similarities  = cosine_distances(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import cluster\n",
    "km = cluster.KMeans(n_clusters=4, init='k-means++', random_state=42)\n",
    "c = km.fit_predict(n_cosine_similarities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "希臘國內三分一自動櫃員機現金短缺\n",
      "歐元區財長拒希臘延長救助計劃\n",
      "呂紹煒專欄：違約與退出 希臘與歐洲才能重生(上)\n",
      "希臘違約在即  歐盟全力穩定經濟\n",
      "希臘脫歐變可能 歐洲衝擊大\n",
      "希債協議  法國願盡最後斡旋努力\n",
      "希臘1／3提款機錢被提光\n",
      "確保銀行穩定 希臘續與ECB緊密合作\n",
      "希臘態度強硬 歐元區耐心漸失\n",
      "希臘盼展延債務 歐元區拒絕\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "titles_ary = np.array(titles)\n",
    "for e in titles_ary[c==3]:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 文章分群(二)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "news = pandas.read_excel('https://raw.githubusercontent.com/ywchiu/pytextmining/master/data/20171214news.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = []\n",
    "titles = []\n",
    "for rec in news.iterrows():\n",
    "    corpus.append(' '.join(jieba.cut(rec[1].content)))\n",
    "    titles.append(rec[1].title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer()\n",
    "X = vectorizer.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_distances\n",
    "n_cosine_similarities  = cosine_distances(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import cluster\n",
    "km = cluster.KMeans(n_clusters=4, init='k-means++', random_state=42)\n",
    "c = km.fit_predict(n_cosine_similarities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "小嫻婚變冒毒菇　勾于美人「奉茶」夢魘\n",
      "明明是好康分享文　網友卻瘋喊噁心慎入\n",
      "「想抱孫要看天！」　他批小嫻婆婆殘忍強逼\n",
      "【好聚不好散】離婚掏出6千萬　這女星比小嫻還慘\n",
      "【狗仔偷拍】小嫻搬離何守正家租66坪房　月租6萬元\n",
      "許聖梅心疼小嫻被當空氣　爆何守正「有兩個女學員」\n",
      "【動畫解盤】毒菇跳火線譙seafood　小嫻難瘦香菇\n",
      "十二月十四日各報頭條搶先報\n",
      "不捨善良小嫻慘遭婚變　乃哥「命運捉弄人」\n",
      "【獨家】小嫻賣房求子　婆婆竟拒入籍何家\n",
      "【毒菇護弟】何守正姊姊不是華岡七仙女　美法連線批小嫻\n",
      "小嫻離婚導火線　拉何守正信妙禪\n",
      "【狗仔偷拍】何守正現身！「全台最沒尊嚴的婆婆」也出來了\n",
      "小嫻中分手魔咒！ 同公司4女星全都婚變\n",
      "他因為小嫻婚變被分手　網友跪求：拜託別放生\n",
      "潔哥目睹「正嫻之變」　驚呼：靠北系列竟是真的\n",
      "何守正姐姐神護航！遭網友酸：全台最討人厭大姑\n",
      "小嫻別傻傻被欺負！女律師說「姐寶」就要這樣對付\n",
      "【小嫻婚變】他說很奇怪　「一定是男生劈腿？」\n",
      "教友小嫻婚姻觸礁　曾之喬談情避不開Seafood\n",
      "小嫻守正結婚在台沒登記　想離婚只有兩條路\n",
      "胡瓜2個月前耳聞小嫻婚變　震驚之餘好心疼\n",
      "昔日搭檔談小嫻婚變　曾國城這樣說\n",
      "何守正兩個姊姊護航扯婆媳　「他」戳破媽寶特色\n",
      "大姑出面護弟！轟小嫻不能生「媽媽是全台最沒有尊嚴的婆婆」\n",
      "小嫻信奉妙禪　關鍵原因與何守正有關！\n",
      "小嫻何守正想離婚　必須先做這件事！\n",
      "【K律師論點】離婚＝失敗？　K律師這麼說\n",
      "真尷尬！他只是聊個天　正妹就退出對話了\n",
      "女網紅因為這理由挺余祥銓！讓人不知該哭還是笑\n",
      "「小嫻不快樂！」　許聖梅：何守正虧欠她\n",
      "一下車有人墜樓掉在車頂　網友：車牌有密碼\n",
      "【內幕動畫】小嫻婚變何守正姊反擊　不滿媽煮飯侍奉星媳婦\n",
      "小嫻多信妙禪？　曾見證「師父帶我跳舞」\n",
      "【話當年】被拍和她上賓館　何守正掰了阿妹\n",
      "小嫻婚變無徵兆　男星嘆：兩人向來出雙入對\n",
      "【獨家內幕】太傷！小嫻被分手　何守正當小三面前攤牌\n",
      "【小嫻離婚】何守正稱沒有遺憾　人妻女星超火「一嘴屁話」\n",
      "【有片】何守正小嫻驚爆離婚　健身房中嗅出端倪\n",
      "【小嫻離婚】3大退讓人財兩失　求子花光430萬積蓄\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "titles_ary = np.array(titles)\n",
    "for e in titles_ary[c==0]:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## silhouette"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import silhouette_score\n",
    "\n",
    "sil_ary = []\n",
    "for k in range(2,21):\n",
    "    km = cluster.KMeans(n_clusters = k, init='k-means++', random_state=42)\n",
    "    c = km.fit_predict(n_cosine_similarities)\n",
    "    sil_ary.append({'group':k,\n",
    "                    'silhouette':silhouette_score(n_cosine_similarities, labels=c)})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'group': 2, 'silhouette': 0.6755654471937644},\n",
       " {'group': 3, 'silhouette': 0.15338953682280074},\n",
       " {'group': 4, 'silhouette': 0.0904743616163152},\n",
       " {'group': 5, 'silhouette': 0.09023331250993877},\n",
       " {'group': 6, 'silhouette': 0.09313344671208439},\n",
       " {'group': 7, 'silhouette': 0.09492595900171423},\n",
       " {'group': 8, 'silhouette': 0.08392193629479042},\n",
       " {'group': 9, 'silhouette': 0.08136447983549083},\n",
       " {'group': 10, 'silhouette': 0.08227220344756503},\n",
       " {'group': 11, 'silhouette': 0.08755001584378343},\n",
       " {'group': 12, 'silhouette': 0.08561332919793142},\n",
       " {'group': 13, 'silhouette': 0.08764196760874017},\n",
       " {'group': 14, 'silhouette': 0.08075272813877192},\n",
       " {'group': 15, 'silhouette': 0.0794831753988302},\n",
       " {'group': 16, 'silhouette': 0.08161712976054764},\n",
       " {'group': 17, 'silhouette': 0.08502936338670943},\n",
       " {'group': 18, 'silhouette': 0.09061744831829563},\n",
       " {'group': 19, 'silhouette': 0.0827405617617394},\n",
       " {'group': 20, 'silhouette': 0.08647846673217109}]"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sil_ary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>silhouette</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>0.675565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>0.153390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>0.090474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>0.090233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>0.093133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>7</td>\n",
       "      <td>0.094926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>8</td>\n",
       "      <td>0.083922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>9</td>\n",
       "      <td>0.081364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>10</td>\n",
       "      <td>0.082272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>11</td>\n",
       "      <td>0.087550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>12</td>\n",
       "      <td>0.085613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>13</td>\n",
       "      <td>0.087642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>14</td>\n",
       "      <td>0.080753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>15</td>\n",
       "      <td>0.079483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>16</td>\n",
       "      <td>0.081617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>17</td>\n",
       "      <td>0.085029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>18</td>\n",
       "      <td>0.090617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>19</td>\n",
       "      <td>0.082741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>20</td>\n",
       "      <td>0.086478</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    group  silhouette\n",
       "0       2    0.675565\n",
       "1       3    0.153390\n",
       "2       4    0.090474\n",
       "3       5    0.090233\n",
       "4       6    0.093133\n",
       "5       7    0.094926\n",
       "6       8    0.083922\n",
       "7       9    0.081364\n",
       "8      10    0.082272\n",
       "9      11    0.087550\n",
       "10     12    0.085613\n",
       "11     13    0.087642\n",
       "12     14    0.080753\n",
       "13     15    0.079483\n",
       "14     16    0.081617\n",
       "15     17    0.085029\n",
       "16     18    0.090617\n",
       "17     19    0.082741\n",
       "18     20    0.086478"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas\n",
    "sil_df = pandas.DataFrame(sil_ary)\n",
    "sil_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages/IPython/core/magics/pylab.py:160: UserWarning: pylab import has clobbered these variables: ['e', 'dist', 'rec']\n",
      "`%matplotlib` prevents importing * from pylab and numpy\n",
      "  \"\\n`%matplotlib` prevents importing * from pylab and numpy\"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1121d1c18>"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEKCAYAAAAcgp5RAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xt0XPV57vHvOxdpJFsz+CLLxoLYpZDGNkaAUGjTFkIokCaBpuniwCkrJISQZpU0bRqfkBScQrvWIZc2OSUsGkJoCA2XFNrGDS6Qk8uB9DSNBTgB2wH7gAEbjGVhS7Iuo7m854/ZkseKZI2kGe2Z4fmsNWv2TXtey6Nn//b1Z+6OiIjUr0jYBYiISGUp6EVE6pyCXkSkzinoRUTqnIJeRKTOKehFROqcgl5EpM6VFPRmdpGZPWtmu8zsuknmf8nMtgav58zsUPlLFRGR2bDpbpgysyjwHPA7wB5gC3C5u2+fYvmPAae7+1VlrlVERGYhVsIyXcAud38ewMzuAy4BJg164HLgs9OtdOnSpb5q1aoSyxQREYAnnnjigLu3zuRnSgn6lcDLReN7gLdOtqCZvQlYDfxgupWuWrWK7u7uUmoUEZGAmb04058p98nYy4AH3D032Uwzu8bMus2su6enp8wfLSIikykl6PcCJxSNtwfTJnMZcO9UK3L329290907W1tntOchIiKzVErQbwFONrPVZtZAIcw3TVzIzH4NWAT8Z3lLFBGRuZj2GL27Z83sWuARIArc6e7bzOwmoNvdx0L/MuA+13OPRd5wMpkMe/bsYWRkJOxS6kYikaC9vZ14PD7ndU17eWWldHZ2uk7GitSHF154gZaWFpYsWYKZhV1OzXN3ent7GRgYYPXq1UfNM7Mn3L1zJuvTnbEiMmcjIyMK+TIyM5YsWVK2PSQFvYiUhUK+vMr5+wwt6HsPj4b10SIibyihBf2Bw+mwPlpE3iCuvvpqtm8v3MS/atUqDhw4wO7du1m3bl1FP3f37t3cc8894+Nbt25l8+bNFf3MYwkt6DO5PLpAR0Qq6Y477mDNmjXz/rkK+oADfcOZsD5eROrM4OAg73rXuzjttNNYt24d999/P+eee+6kj1rJ5XJ8+MMfZu3atVxwwQUMDw8DhUA+++yzWb9+Pe9973s5ePAgwFHrOXDgAGPP6crlcmzYsIGzzjqL9evX89WvfhWA6667jscff5yOjg4+97nPsXHjRu6//346Ojq4//77GRwc5KqrrqKrq4vTTz+d73znOxX93ZTyrJuKebVvhOOaG8IsQUTK7MZ/28b2V/rLus41xyf57HvWHnOZhx9+mOOPP56HHnoIgL6+Pm677bZJl925cyf33nsvX/va17j00kt58MEHueKKK3j/+9/PLbfcwjnnnMPGjRu58cYb+fKXvzzlZ379618nlUqxZcsW0uk0b3vb27jgggu4+eab+eIXv8h3v/tdANra2uju7uYrX/kKAJ/5zGc477zzuPPOOzl06BBdXV2cf/75LFiwYDa/nmmFetXNvn7dXCEi5XHqqafyve99j0996lM8/vjjpFKpKZddvXo1HR0dAJx55pns3r2bvr4+Dh06xDnnnAPAlVdeyWOPPXbMz3z00Uf55je/SUdHB29961vp7e1l586d09b66KOPcvPNN9PR0cG5557LyMgIL7300gz+tTMTaot+X5+CXqTeTNfyrpRTTjmFJ598ks2bN3P99dfzjne8Y8plGxsbx4ej0ej4oZupxGIx8vk8wFHXtrs7t9xyCxdeeOFRy//oRz865vrcnQcffJA3v/nNx1yuXMJt0SvoRaRMXnnlFZqbm7niiivYsGEDTz755Ix+PpVKsWjRIh5//HEA7r777vHW/apVq3jiiScAeOCBB8Z/5sILL+S2224jkymcb3zuuecYHBykpaWFgYGB8eUmjl944YXccsst4xekPPXUU7P4F5cutKCPRUxBLyJl8/TTT9PV1UVHRwc33ngj119//YzXcdddd7FhwwbWr1/P1q1b2bhxIwCf/OQnue222zj99NM5cODA+PJXX301a9as4YwzzmDdunV85CMfIZvNsn79eqLRKKeddhpf+tKXePvb38727dvHT8becMMNZDIZ1q9fz9q1a7nhhhvK9nuYTGjPujnuxF/zS/7ym9x1VVcony8i5bNjxw7e8pa3hF1G3Zns91pTz7qJRyNq0YuIzIMQg9501Y2IyDwI7xh9NELfcIbh0Ul7HRSRGqM73curnL/PUFv0oGvpRepBIpGgt7dXYV8mY8+jTyQSZVlfaNfRx6MRcsCrfcOsXlqZu8FEZH60t7ezZ88eenp6wi6lboz1MFUOoQb9CPCaWvQiNS8ej/9ST0hSPUI/dPOqrrwREamo0II+YkZLIsZrCnoRkYoK9REIK1IJtehFRCos1KBvSyZ0jF5EpMLUohcRqXOhBv3yZIKew2kyuXyYZYiI1LVwgz7VhDv0DKijcBGRSikp6M3sIjN71sx2mdl1UyxzqZltN7NtZnbPZMtMtDxVePi/7o4VEamcaW+YMrMocCvwO8AeYIuZbXL37UXLnAx8Gnibux80s2WlfPjyZBOgDkhERCqplBZ9F7DL3Z9391HgPuCSCct8GLjV3Q8CuPv+Uj58earwHAcFvYhI5ZQS9CuBl4vG9wTTip0CnGJm/2FmPzGziyZbkZldY2bdZtbd09PDouY4DbGIDt2IiFRQuU7GxoCTgXOBy4GvmdlxExdy99vdvdPdO1tbWzEzlicTatGLiFRQKUG/FzihaLw9mFZsD7DJ3TPu/gLwHIXgn9bylIJeRKSSSgn6LcDJZrbazBqAy4BNE5b5VwqtecxsKYVDOc+XUsDyZEKHbkREKmjaoHf3LHAt8AiwA/i2u28zs5vM7OJgsUeAXjPbDvwQ2ODuvaUUsCJVCHp1WCAiUhklPY/e3TcDmydM21g07MAngteMtCUTjGbzHBzKsHhBw0x/XEREphHqnbFQaNFDoacpEREpv9CDvi0Iej3FUkSkMkIP+iMtegW9iEglhB70rQsbiRjqaUpEpEJCD/pYNEJrS6Na9CIiFRJ60IOupRcRqaTqCHrdHSsiUjHVEfRq0YuIVEx1BH2qiYGRLIPpbNiliIjUnSoJevU0JSJSKdUR9OppSkSkYqoj6NXTlIhIxVRH0CeDoNehGxGRsquKoG9qiJJqiqtFLyJSAVUR9FB45o3ujhURKb+qCfq2ZEJPsBQRqYCqCXq16EVEKqNqgr4tmaB3MM1oNh92KSIidaVqgn5FKoE77B9Qq15EpJyqJujV05SISGVUTdCrpykRkcqomqAfv2lKQS8iUlZVE/SppjiJeERBLyJSZlUT9Gam59KLiFRASUFvZheZ2bNmtsvMrptk/gfMrMfMtgavq2dTjHqaEhEpv9h0C5hZFLgV+B1gD7DFzDa5+/YJi97v7tfOpZjlyQTdLx6cyypERGSCUlr0XcAud3/e3UeB+4BLKlHM8lQTr/WPkM97JVYvIvKGVErQrwReLhrfE0yb6H1m9nMze8DMTphsRWZ2jZl1m1l3T0/PL81fnmwkk3NeHxotpXYRESlBuU7G/huwyt3XA98D7ppsIXe/3d073b2ztbX1l+YvT6mnKRGRcisl6PcCxS309mDaOHfvdfd0MHoHcOZsilFPUyIi5VdK0G8BTjaz1WbWAFwGbCpewMxWFI1eDOyYTTHjd8fqEksRkbKZ9qobd8+a2bXAI0AUuNPdt5nZTUC3u28C/sTMLgaywOvAB2ZTzNKFjUQjxmtq0YuIlM20QQ/g7puBzROmbSwa/jTw6bkWE40Yy1oa9bwbEZEyqpo7Y8eopykRkfKquqAv9DQ1HHYZIiJ1o+qCvtCiT0+/oIiIlKTqgn5FKsHhdJaBkUzYpYiI1IWqC/rl6mlKRKSsqi/ok+ppSkSknKov6HV3rIhIWVVd0LepS0ERkbKquqBPxKMsao6rpykRkTKpuqCHwlMs1aIXESmP6gz6ZKNa9CIiZVKdQa8WvYhI2VRn0CcT9A6Oks7mwi5FRKTmVWXQjz2Xfr8ehSAiMmdVGfRtY9fS6zi9iMicVWXQj/c0peP0IiJzVpVBP3bTlHqaEhGZu6oM+mQiRnNDVC16EZEyqMqgNzOWq6cpEZGyqMqgh8LDzdTTlIjI3FVv0KunKRGRsqjeoE8VDt3k8x52KSIiNa2qgz6bdw4MqlUvIjIX1Rv0ei69iEhZlBT0ZnaRmT1rZrvM7LpjLPc+M3Mz65xrYeppSkSkPKYNejOLArcC7wTWAJeb2ZpJlmsBPg78VzkKW67HIIiIlEUpLfouYJe7P+/uo8B9wCWTLPdXwOeAsiTz0gWNxCKmFr2IyByVEvQrgZeLxvcE08aZ2RnACe7+UNkKixhtyYRa9CIiczTnk7FmFgH+FvjzEpa9xsy6zay7p6dn2nW3JRvVohcRmaNSgn4vcELReHswbUwLsA74kZntBs4GNk12Qtbdb3f3TnfvbG1tnfaDV6Sa1KIXEZmjUoJ+C3Cyma02swbgMmDT2Ex373P3pe6+yt1XAT8BLnb37rkW15ZMsK9vBHfdNCUiMlvTBr27Z4FrgUeAHcC33X2bmd1kZhdXsrgVqQRDozkG0tlKfoyISF2LlbKQu28GNk+YtnGKZc+de1kFbUXX0icT8XKtVkTkDaVq74yFIz1N6YSsiMjsVXXQ6zEIIiJzV9VBvyzZCOjuWBGRuajqoG+MRVmyoEFdCoqIzEFVBz0ceS69iIjMTvUHfTKhFr2IyBxUf9CrRS8iMifVH/TJBK8PjjKSyYVdiohITar+oA+upd+vjsJFRGalZoL+1b7hkCsREalNVR/0K9TTlIjInFR90Lfp7lgRkTmp+qBvScRZ2BhTi15EZJaqPuhBPU2JiMxFTQS9epoSEZm9mgj6sZ6mRERk5moi6FekEuwfSJPLq0tBEZGZqomgb0slyOWdA4d105SIyEzVRNCv0CWWIiKzVhNBf+TuWAW9iMhM1VTQ6ymWIiIzVxNBv7i5gXjU1KIXEZmFmgj6SMRoS+q59CIis1ETQQ9jPU3pCZYiIjNVUtCb2UVm9qyZ7TKz6yaZ/0dm9rSZbTWzH5vZmnIXWuhpSpdXiojM1LRBb2ZR4FbgncAa4PJJgvwedz/V3TuAzwN/W+5Cx1r07rppSkRkJkpp0XcBu9z9eXcfBe4DLilewN37i0YXAGVP4+WpBCOZPP3D2XKvWkSkrsVKWGYl8HLR+B7grRMXMrM/Bj4BNADnlaW6IuPX0vcPk2qOl3v1IiJ1q2wnY939Vnc/CfgUcP1ky5jZNWbWbWbdPT09M1r/eE9TusRSRGRGSgn6vcAJRePtwbSp3Af83mQz3P12d+90987W1tbSq0Q9TYmIzFYpQb8FONnMVptZA3AZsKl4ATM7uWj0XcDO8pVYsKwlgZn6jhURmalpj9G7e9bMrgUeAaLAne6+zcxuArrdfRNwrZmdD2SAg8CV5S60IRZhyQL1NCUiMlOlnIzF3TcDmydM21g0/PEy1zWpFamEWvQiIjNUM3fGgnqaEhGZjZoKerXoRURmrqaCfnkqwaGhDCOZXNiliIjUjNoKel1iKSIyY7UV9OppSkRkxmoy6PVcehGR0tVW0CfVohcRmamaCvoFjTFaEjG16EVEZqCmgh7U05SIyEzVXtCnEuxTT1MiIiWrvaBPJtinFr2ISMlqLuhXpBL0DKTJ5vJhlyIiUhNqLujbUgnyDj2HdfhGRKQUNRf06mlKRGRmai7o1dOUiMjM1FzQr0g1AeppSkSkVDUX9Iua4zTEImrRi4iUqOaC3swKl1iqRS8iUpKaC3oYuztWQS8iUoraDPpUQs+7EREpUc0G/at9I7h72KWIiFS92gz6ZILRbJ5DQ5mwSxERqXq1GfTqaUpEpGQ1HfQ6Ti8iMr2Sgt7MLjKzZ81sl5ldN8n8T5jZdjP7uZl938zeVP5Sj1BPUyIipZs26M0sCtwKvBNYA1xuZmsmLPYU0Onu64EHgM+Xu9BirS2NREx3x4qIlKKUFn0XsMvdn3f3UeA+4JLiBdz9h+4+FIz+BGgvb5lHi0cjLF3YqOfSi4iUoJSgXwm8XDS+J5g2lQ8B/z6XokqxQj1NiYiUJFbOlZnZFUAncM4U868BrgE48cQT5/RZbckEu3sH57QOEZE3glJa9HuBE4rG24NpRzGz84G/AC5290mb2u5+u7t3untna2vrbOodtyKV0IPNRERKUErQbwFONrPVZtYAXAZsKl7AzE4Hvkoh5PeXv8xf1pZK0D+SZWg0Ox8fJyJSs6YNenfPAtcCjwA7gG+7+zYzu8nMLg4W+wKwEPgnM9tqZpumWF3ZqKcpEZHSlHSM3t03A5snTNtYNHx+meuaVnFPU7/SunC+P15EpGbU5J2xoJ6mRERKVbNBr7tjRURKU7NB39QQJdUU1/NuRESmUbNBD+ppSkSkFLUd9OppSkRkWrUd9GrRi4hMq7aDPpXgwOE0mVw+7FJERKpWzQe9O+wf0MPNRESmUvNBD7o7VkTkWGo76JMKehGR6dR00LcvaqIhGuEff/Iio1kdpxcRmUxNB31LIs7N7zuV/3y+lw0P/Ix83sMuSUSk6pS145Ew/P4Z7ezrH+HzDz9LWzLBZ373LWGXJCJSVWo+6AE+es5J7Osb4fbHnqctmeBDv7k67JJERKpGXQS9mfHZ96xlf3+av35oO23JRt69/viwyxIRqQo1fYy+WDRifPmyDjrftIhP3P8z/vP/9YZdkohIVaiboAdIxKN87f2dnLikmWvu7uYX+/rDLklEJHR1FfQAxzU3cNdVXTQ3RPnAnVt45dBw2CWJiISq7oIeYOVxTXzjg10MprN84B9+St9QJuySRERCU5dBD/CWFUm++v4zeeHAIB++u5uRTC7skkREQlG3QQ/wGyct5W8u7eCnL7zOJ769lZxuqBKRN6C6uLzyWC4+7Xj294/w1w/tYFnLdj77njWYWdhliYjMm7oPeoCrf+tX2Nc3wh0/foEVqQQfOeeksEsSEZk3b4igB/jM776Fff0j/M9//wXLko289/T2sEsSEZkXJR2jN7OLzOxZM9tlZtdNMv+3zexJM8ua2R+Uv8y5i0SMv7n0NM7+lcVs+Kef8/jOnrBLEhGZF9MGvZlFgVuBdwJrgMvNbM2ExV4CPgDcU+4Cy6kxFuX293fyq8sW8kd3P8Eze/vCLklEpOJKadF3Abvc/Xl3HwXuAy4pXsDdd7v7z4Gqfyh8MhHnGx/sItUU54Pf2MLLrw+FXZKISEWVEvQrgZeLxvcE02rW8lSCu67qIp3JceU//JSDg6NhlyQiUjHzeh29mV1jZt1m1t3TE+4x8pPbWrjjyrPYc3CYD921heFR3VAlIvWplKtu9gInFI23B9NmzN1vB24H6OzsDP3upa7Vi/m7yzr46Lee5GP3PsXfX3EGsej830OWzzuDo1kG0zkOpzMcTucYTGc5nM4ymM6SzuZJZ3KMZPOkM3nS2VxhWjZHOpMPphdNC5YbCeaPTV/YGGPt8SnWrUyy7vgUp7anWNbSqPsKROpcKUG/BTjZzFZTCPjLgP9e0arm0UXrVvCX71nLZzdt49p7nuLU9hQAZmAYZhApGi7MM2x8mWB8wnA6kx8P6uL3wnAhyAeC6UMz3JuIRYxEPEpjLFJ4jQ0H7wsbYyxZECURj9AYi9IYLyz3+uAo217p5/u/eA0PNrNLFzaOB/+6lYWNwMrjmhT+8oYyms3TczjNa/0j7O9Ps39ghAOHR1myoIE3LWlm1ZIFrFzURDyEhmA5TBv07p41s2uBR4AocKe7bzOzm4Bud99kZmcB/wIsAt5jZje6+9qKVl5GV/7GKnoHR/nKD3by8LZ9ZV33WPAuCF4LG6MsXVj48iwcnxYrWiY6YTxG03iQR2iIRua81zGYzrLj1X6e2dvHM68U3h/feWD8ERHHNcePCv51x6c4cXEzkUj9hf9gOkvPQJqew+nC+0Cag0OjxKOFjePYBjURjwavYDjYgCZiwQY1mNcQjUy5kXT3o/a2hkdzjGRzjGTyjGRyDGdyhT23ovGx4dFcnsXNDbQvamLloibaFzWzqDmuDfI00tkcPQNpXutP0zMwwmtBiBfe0+zvH2H/QJrXSzhPF40Y7YuaeNOSBaxa0nzU+wmLm2iMRSv278jnnYGRLH3Ds3tAo7mHcwSls7PTu7u7Q/nsqWRzefIOjuNO4TU2TOEPtfAeDE82ncLEhliEBY2xmmkBjGRy/GLfAM/s7WPbK308vbePZ/cNkMkVvh8tjTHWBqG/dmWSpQsbaW6I0dwQZUFDjObGKM0NUZri0dDDJ5PL03t4NAjwkfEAnxjoPQNpBst8bsaMIxuGWBSzwu92JAj32f65xaM2/n8xpikeLQr+JlYe13zU+NIFjXWxcR7bQPaPZOgfzjIwkqF/JEv/cIaBkSz9I5nCtOHC8OuDo+Ot8oOTPLk2GjFaFzayLNnIspYEy5KNtI29F01b3NzA64Oj7O4dYnfvIC/2DrK7d4iXeofYfWCQgXR2fJ1mcHyqiVVLmydsCBZw4uJmmhqi42F9aHiUvuHMpK/+yaYPZRhIZ8e/Oy9+7t1PuHvnTH6HCnqZ0mg2z3OvDbDtlT6e2dvP03v72PFqP+ns1FfRmkFzPEpTQ2HvZGxjML5BaIjS3FgYbmqI0hiLksvnyeadXN6PvOec7Nj03Nj0I8tlckePZ3NO33CGnsNTt86SiRjLkglaFzbS2lL0mjC+qLmBbD7PSKboPEhRWI9kcuOt8pHxebmjzqWMTXMvdIjT1BAlERtr+Rc2iON7B/GiPYZgD6GwfHR8j8IM+oez7Dk0xJ6Dw+w9OFx4PzTE3kOF4UMTQq0hFmHlcWMbgabxjUBbS4KGWIR4tPBqiBnxYE8xHjUaokfmxaM2ow13Lu/BnkiwxxL83oaDPZTh0RzpYG9muGivZXg0Ox7axWE+FuCjuWNfuR2NGMlEjJZEnEXNcZYlEyxraaSt6L01eF+8oIHoHDeA7s7BocyRDcCBofENwYu9g7+0gWlpjHF4NHvMDX1DNEKyKU6qKUaqKf5Lr2TwfulZJyropbKyuTwvHBjk0HCGodEcQ+ksg6M5hkcL70PBOYfB0RxDo4XhoeBE8/BojsHR7Pj7SOboP96IQSwSIRoxYhEjGi28j0+L2pF5kUjwbsSjRsSMZFOcZVME+NKFjSTildu1rgaH01n2BuFfvDHYc6gwfOBwelbrjUVsPPTHNhCxaGFaLu9HBfp0gTwZM0jEoiSbCkE9FtjJpuLhI/OSwXgyER+fVw17ksX6hjK8+PrYHsAgBw6PFmqfEN7HNTeMDyfiUx/2K2ZmCnqpHYWWeZ5oxIia1cVhhmo2ksmx99Aw+/vTZHL54OVFw3lGc062aN5otjCczR8ZHp+XyxOLWLB3UthrGdtTKZ6WiAXvRXsyR5Yd22PR/32pZhP0b5iHmkn1iUaMaKS+W9nVJBGPclLrQk5qXRh2KTLPauNMoYiIzJqCXkSkzinoRUTqnIJeRKTOKehFROqcgl5EpM4p6EVE6pyCXkSkzoV2Z6yZDQDPhvLhU1sKHAi7iAmqsSaozrpUU2lUU+mqsa43u3vLTH4gzDtjn53pbbyVZmbdqqk01ViXaiqNaipdNdZlZjN+dowO3YiI1DkFvYhInQsz6G8P8bOnoppKV411qabSqKbSVWNdM64ptJOxIiIyP3ToRkSkzs170JvZCWb2QzPbbmbbzOzj813DVMwsamZPmdl3w64FwMyOM7MHzOwXZrbDzH69Cmr6s+D/7Rkzu9fMEiHVcaeZ7TezZ4qmLTaz75nZzuB9URXU9IXg/+/nZvYvZnZc2DUVzftzM3MzW1oNNZnZx4Lf1TYz+3zYNZlZh5n9xMy2mlm3mXXNc02TZuVsvudhtOizwJ+7+xrgbOCPzWxNCHVM5uPAjrCLKPK/gIfd/deA0wi5NjNbCfwJ0Onu64AocFlI5XwDuGjCtOuA77v7ycD3g/Gwa/oesM7d1wPPAZ+ugpowsxOAC4CX5rkemKQmM3s7cAlwmruvBb4Ydk3A54Eb3b0D2BiMz6epsnLG3/N5D3p3f9XdnwyGByiE18r5rmMiM2sH3gXcEXYtAGaWAn4b+DqAu4+6+6FwqwIK9140mVkMaAZeCaMId38MeH3C5EuAu4Lhu4DfC7smd3/U3bPB6E+A9rBrCnwJ+B/AvJ+km6KmjwI3u3s6WGZ/FdTkQDIYTjHP3/VjZOWMv+ehHqM3s1XA6cB/hVlH4MsUvvgz7924MlYDPcA/BIeT7jCzBWEW5O57KbS0XgJeBfrc/dEwa5qgzd1fDYb3AW1hFjOJq4B/D7sIM7sE2OvuPwu7liKnAL9lZv9lZv/HzM4KuyDgT4EvmNnLFL738703Nm5CVs74ex5a0JvZQuBB4E/dvT+sOoJa3g3sd/cnwqxjghhwBnCbu58ODDL/hyKOEhwLvITCRuh4YIGZXRFmTVPxwuVkVXNJmZn9BYVd8W+FXEcz8BkKhyKqSQxYTOEQxQbg2xZ+j+EfBf7M3U8A/oxg73q+HSsrS/2ehxL0ZhanUPi33P2fw6hhgrcBF5vZbuA+4Dwz+8dwS2IPsMfdx/Z2HqAQ/GE6H3jB3XvcPQP8M/AbIddU7DUzWwEQvM/r7v9UzOwDwLuBP/Twr2c+icKG+mfB970deNLMlodaVeH7/s9e8FMKe9bzepJ4EldS+I4D/BMwrydjYcqsnPH3PIyrbozClnGHu//tfH/+ZNz90+7e7u6rKJxc/IG7h9pSdfd9wMtm9uZg0juA7SGWBIVDNmebWXPw//gOquvk9SYKf5wE798JsRYAzOwiCocEL3b3obDrcfen3X2Zu68Kvu97gDOC71uY/hV4O4CZnQI0EP7DxF4BzgmGzwN2zueHHyMrZ/49d/d5fQG/SWFX4+fA1uD1u/NdxzHqOxf4bth1BLV0AN3B7+pfgUVVUNONwC+AZ4C7gcaQ6riXwnmCDIWw+hCwhMJVCDuB/w0sroKadgEvF33X/z7smibM3w0sDbsmCsH+j8H36kngvCqo6TeBJ4CfUTg2fuY81zRpVs57RxVnAAABy0lEQVTme647Y0VE6pzujBURqXMKehGROqegFxGpcwp6EZE6p6AXEalzCnoRkTqnoJc3tODhbCJ1TUEvdc3MbjCzZ83sx8Hz8z9pZj8ysy+bWTfwcTNbZWY/CJ4Z/30zOzH42W+Y2R8Uretw8H6umT1mZg8F6/57M9PfklQtfTmlbgVPQHwfhWf5vxPoLJrd4O6d7v43wC3AXV54Zvy3gL8rYfVdwMeANRSeH/P75axdpJwU9FLP3gZ8x91HvPA8738rmnd/0fCvA/cEw3dTuPV8Oj919+fdPUfh9vlSfkYkFAp6eaMaLGGZLMHfSHBopqFo3sRnh+hZIlK1FPRSz/4DeI+ZJYJner97iuX+L0e6RPxD4PFgeDdwZjB8MRAv+pkuM1sdbAD+G/DjchYuUk664kDqlrtvMbNNFJ7+9xrwNNA3yaIfo9CT1wYKvXp9MJj+NeA7ZvYz4GGO3gvYAnwF+FXgh8C/VOQfIVIGenql1DUzW+juh4OelR4DrvGgH845rPNc4JPuPtUegkhVUYte6t3tZrYGSFC4smZOIS9Si9SiFxGpczoZKyJS5xT0IiJ1TkEvIlLnFPQiInVOQS8iUucU9CIide7/A1gsr3DzMnSEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "% pylab inline\n",
    "sil_df.plot(kind = 'line', x='group', y='silhouette')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 語音辨識\n",
    "- speech_recognition\n",
    "- pyaudio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'好今天我來到這邊上課'"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import speech_recognition\n",
    "r = speech_recognition.Recognizer()\n",
    "\n",
    "with speech_recognition.Microphone() as source:\n",
    "    r.adjust_for_ambient_noise(source)\n",
    "    audio = r.listen(source)\n",
    "r.recognize_google(audio, language='zh-TW')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'4 kn'"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import speech_recognition\n",
    "r = speech_recognition.Recognizer()\n",
    "\n",
    "voice = speech_recognition.AudioFile('/Users/davidchiu/Downloads/out5130716119403203006.wav')\n",
    "with voice as source:\n",
    "    audio = r.record(source)\n",
    "\n",
    "captcha = r.recognize_google(audio, language='en')\n",
    "captcha"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 社群偵測法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "news = pandas.read_excel('https://raw.githubusercontent.com/ywchiu/pytextmining/master/data/20171214news.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = []\n",
    "titles = []\n",
    "for rec in news.iterrows():\n",
    "    corpus.append(' '.join(jieba.cut(rec[1].content)))\n",
    "    titles.append(rec[1].title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer()\n",
    "X = vectorizer.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "n_cosine_similarities  = cosine_similarity(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = (n_cosine_similarities > 0.4).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "G = nx.from_numpy_matrix(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeEAAAFCCAYAAADGwmVOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXuQHNV59p+e2cvMUAmWMQJMhICKy+hiSRitgmNXDB9lhKSVzS5QBYU2FlUheBMINiDJsmMjIInYNWDA9m5sqhKZFCXigiCDAEnEODiJjXdkIy1IOIULU1xiNLLBxEar2+p8f7zp7Gg0O9uX033O6X5+VV3S7s50n+k5fd7z3j2llAIhhBBCUqdgegCEEEJIXqEQJoQQQgxBIUwIIYQYgkKYEEIIMQSFMCGEEGIICmFCCCHEEBTChBBCiCEohAkhhBBDUAgTQgghhqAQJoQQQgxBIUwIIYQYgkKYEEIIMQSFMCGEEGIICmFCCCHEEBTChBBCiCEohAkhhBBDUAgTQgghhmgzPQBCSARqNWDDBmB0FHjnHeD444F584CrrgJOPNH06I7FtfESkhKeUkqZHgQhuSOqUKpWgfXrgSeflJ/375/4W7kMKAUsWQKsXQt0dUW7vlJTj63Z+888U/728ssTv5s2TX5++ul44yUko1AIEzIVQQRmUKEaR4gODwM33QSMjcnrJsPz5Fx33AH09x/9t1bX7+wEDh2S/7e1AQcPTvytVAIOHwbe+15gfBx4+22gUJDf6aBYBC68EFi4cEKId3YC+/YBlQpw4EC8+06IpVAIEzIZQQTmuefKz88+2/w1R44Af/iHwIwZwOuvAy++KEJsKopFYPly4FvfEmHiC+B9+4KPv1gEZs0C5s8XwfTuu8Df/Z0+wWmCoPedGjZxBAphQpoRVOtMmrY24LTTgFdeEYEeFc8z+zlM0MoiQIglUAiT7DOVybLx73v2ADt2TJhmidtUKuEEMU3cJEUohEl2mcqcPD4OnHAC8Otfi3+z/u8kWxQKwB//MTBz5uQBaIcPA6+9Bvz856JF08RNUoBCmGQTW8zJxD5aBaBNBU3cRDMUwiR7DA8DN94oApiQJAhr4iZkEiiEids0+u9+/WuJmOW0JknT3g5ccIFo0/Qbk4hQCBPzhA2cmqoIBCEmaExJo3AmAaAQJuZg4BTJAwzqIi2gECbpMFka0OHDNB2T/NBYQIUacu6hECbJ0krbJSTPUEMmoBAmOmmm7T73nNtlEglJGj/t6eab5WcWCckVFMIkPtR2CdFDW9vRm1Zqy5mHQpjEg0UxCEkeFgnJLAXTAyAOU9/ZhwKYkORQSp6z668HvvIV06MhGqEmTKamWZ5uZydw//309xJigvPOAwYHaZ7OABTCZHLo6yXEXsplYPFi4LjjGMjlMBTCRGBkMyHuw0Au56AQzhuNwrZV+zZCiJswkMsZKITzAk3LhOSPSoX5x5ZDIZwHmEZESL7p6Di6bzLN1tZAIZx16tOICCGkHpqtjUMhnGWqVUlloAAmhLSiUqEgNgSFcJbp7QU2baIJmhAyNZUK8MwzwMKFpkeSKyiEs0R95HOtBvzrv1IAE0KCM2MGcO21DNpKEQrhLMDIZxIHz+NmjUxQKsm/S5YA11wD7NzJyOoEoRB2HUY+k0Y8DygUgCNHWs+J+qCc3/4W+OIXWZyFHAs7OyUKhbBr1Jucd+4EXnwRGB83PSpiC8UisHw58Bd/Afz93wNPPCHCdmxs4jX+Irp0qSyivg+QGzoSBkZWa4FC2BVocs4vnicNM846C/jZz44Vqo3UmxM/8xnZrD3/PPD228C0acCHPgSsXHmsObFWA/7mbySY74035Dr1G7xyWbTrQ4fkX0IARlbHRRH7GRpSqlJRyvOUEh2FRx6OclmpUkmp3l6lqlWZC7WaUh//eLD3e57Mm6Gh1vNrZESpnh6lOjuVKhaPPYfnKTVjhlLXXSfX7+mJNxfnzlVqYEA+n+l7zEPPUalMzFESCmrCtsNiG/mhrQ348IeB6dMn11ivuAJ48MFw522lqQwPA5/7HHDgwNTn6ewEvvpVMV9HzT+vT4Oh+Ts7eB7Q0wM8/LDpkbiH6V0AacHIiOwwTe9yeaSjSUylsa5eHe/8jZrK0JBS7e3hztPeLu/zrTNxP2O1Kpp+qXSsZtzZqVShIEdn59F/K5eVamuTv5n+7njIUSqJpYSEgpqwbdQHXn3ve8Cbb5oeEUmSQgG4+OKjA6SaUa0C554b3RfbqKlUq8BHPyr+3bC0twM//KGcI4gmGySAZ+9emffNfNfA5H976CFq07ZQLEp/4w0bmL4UBtO7APK/+H65UkkO07taHsGPRj9q0KO9Xalt24LNj6VL44+zXlM577x45zr/fDlPK022mU87CVqNgUe6R7Eo30NPj6xpZEqoCdsAfWPuUqkA110HfO1r4XykYSJKazXg/e+Pn4pWKgG33gp8+tPAKafEi3AuFMRK42s8rTTZtLSiZmP43e+ALVtaR5MT/TB9KTimdwG5J4pvjYcdR72PM2gEe9CI5XoGBvT5Pvv6lPrSl/Sc68tfTuaZ0E2Y76azU3zNpudWVo6wcz2HFExvAnKN71Nj5HN6FAqyQ6+nXBYtccEC+dfzWp/D847VZPv7Jeq3p0fOMdk1enrkdWG0g9FRfXm5b78NPPaYnnNt3qznPEkT5rv5j/8A7r1XItVJfPbtkzVu+3bTI7EWmqNNwi5H6VKpyP3esWNys+n27VIUJUylqUZ0m2aXL9cn8Pr6gKeflmIccTn1VOD11+OfJ02CfjeLFskmmcSH6UstoRA2Ra0GzJzJyldpEbaqjw0+Tp8VK4AHHoh/nmJRNhj33JNfIRwUnRsfIpaGV19l1HQTaHMxxYYNLJavg6k6AEUNEDnxRGDVqvjj08G8eZKKE6SgRis8TzYRGzfqEcInnRT/HLZy/PGmR5AtDh4EbrtNTP3kKCiE06I+//eddyTPkkI4PH5d5KVLgU99Cvjud4OZjk87DRgcdLMl28qVwM03xz/P4sXyWbu7geeei3++5cvjn8NW5s0T8yktVXo4cgQYGgJmzWK0dAM0RycNGy/oY8YMSQdqNAm3Mh2/8gqwbh2wdasI5foAp/pGB7a3ZOvtBR55JPr7i0Xg2WfFj12r6U9Ryhp0FyWD3+XrW9/K7twJi8nQ7MzDxgt6Ux3CFnxYvTp4ao/tqRRxS5iuWXP0+XQV68gycRtV8Jj86OhgQY//BaYHkFmY/6vviCIgL788neukSdQ5dfnlx55rZEQWwqgLaB465rB2e7JHlJz5DEJzdBJUq9G7zJAJogZVrVkj/t8o1Hf5sZEwXbUKBQkuu/32yc/12c9K0ExQOjqAu+8249drjKvo7JT7UKlI0FoSfv4oXcyKRZm77e2s1BWEvPcjNr0LyCQ0Y+k5li0Lr3GNjESv5ewfvb3JzAtd1NdKbqwzXizKEfTeDQ0Fr1VeKpnRWsLUVffrVes0dUaphlarKTU4KBXKTjzR/LNk+1EsTvSrzhkwPYDMsWcPGzDoOAqFaAv+kiXxr+1KS7b6hb67W/4dHAw/dl+oNyvZ2NYmv0+6CcNkRI2r0G3qjNOoYmSELReDPvM5bP5Ac7RuBgclnYRRlfEJaxqu1aSARNzUL7/RQdA84UYzqUvpT/XYVKAEiGYKbkS3qTPqPYrjIskbeWv+YHoXkDmuvNL8jjIrh+eFMw0PDMQ3RftHX9/U19uyRak5cyYaz9e/PwmzaJ7QGRQVJbI+CaIEC+b5yEnQFhs46Oadd0yPIDsoJYU49u4N9vrR0fjt/nzefnvyv1Wr0uzhoouAXbsk37Yx53ZsTKwhmzZJkN7wsJ5x5YX16/UFNY2NyflMs3GjaMTFoumRuEFOmj9QCOtmzx7TI8gWnifmvyDo3ABNm9b898PDwMc+BuzcGew8Sk0sJhTEwajVpLiNUnrOF3YzlyS33y5FU5YtY6emINiygUoQCmGdVKvSoYfoY2xM/G9B0FXvt1gUP18jw8PADTeES+nxycmuXgtBN11hCLOZS5qFC6U5xM6d1IqnwqYNVEJQCOtk/XrWg06CVqbheubNmyhFGZeVK4/+2e/9HCfgLge7ei2MjuoPbAyzmUuLzZsll5i0xm/+kFEohHWh24RGJpjMNNxIo+CMykUXHRvpqsNHmYNdvRaSiqsIuplLiyQ2G1nEb/6QUXcOhbAuNmyIVxCfNKdcbm4absb06dKMwfOiX69QkIYP9ejcYNlkFrWVpNoIBt3MpQWDOIMzPp7ZuAoKYV2MjkbzFWaVgqappVQ4DXftWhHcUVm16ti8ZJ1C00azqG3odCv4hNnMpQV7Focjo3EVFMK6qNVMj8AOPE8KJNx0k/wb91xLl4YrFNHVJUn+Ua59+eXN6yzrNhvaZha1DV1uhXrGx5M5bxyS2GxknQzGVVAI6+LFF02PIF0aF49yWX7X0yNVrgYGRBh2dka/Rrksmm1Y+vsnBHEQ03SxKPmbGzc2/7tus6FtZlHb8N0KOhkfBx56SO8542LbpsAFlAIefTRT6y2FsA6qVeD1102PIj3OOkvKOvb1Ad3d8u8ttwCvvgo8/PCEObe/H/jqV6NFgPrlBqN2M+rvl81AT49sDho3DcWi5GkuWyZ5m5N1GgL0mg1tNIvayNq18S0p9djoU9QRw5BHDh+WZ6i3V9Zex2HtaB0sWBC8eEMWWLoUePzx4K/fvh24+upgOdRJ1I2NWxNZZz3wUkk2Ky7VlDbF8DDwuc9Jm0Jd2Naqkm1Po5OVGtOm62Y6z8CA+RqraR/d3dHu1bZtSs2d27rWsqluPa3Q2RnL9jaJtjE0pFR7u765G7YeeRr4naLCfI72dqU6OtgyFXC+xjQ14ThUq8BHPwocOmR6JOnS1wfcf3/099vWrScIvb1SBzrO42KbFuYKYSwpQejoEPeRTXPN7xg1NtZ6jtVrf11dEqT0xBOyBumqm+4iLj9bpncBTtPTY34XmPZRLkvP2rwRt6tPZ6fTu3Ur2LZNqZNOij+HCwU753DUnsW1mlJLl+a7Z7GNFo6AUBOOyq5dwPz5+dt95tmnGbW/bXs7cM89bvutbOHSSyX4Ly6XXQZ85zvxz5MEUSxFtRowc2a+K3A5ujaxjUdYqlUxAT32WP4EcJS83SzhC9EgZkOfBQuA++5z00yWFLWaCJnRUUn/Ov54yZm96qqp59bu3XrG8MILes6TBCeeKEVjwuBHWsd1mbiMX40u7L0zjWlV3Cn8AIq8BkPY0hzdNK3Mhn7Q2dy5Yj4lE4yMiAunVDo20M03t/b0yOsm4/TT9czlM85I73OnRVyXSRaOvj7T30JoaI4OSlRTZFbw83ZpUp3AxQAzUwQNPAIkh3v5cuBb3zr2Ps6aBfzsZ/HHM2uWPq3aJvK+TnV3i5XSIWiODoLfxi6PEzsruXhJEMVsmEfCCobxcTGrbt4swnjtWokEBoA5c/QI4blz45/DRvxn9Npr89lQxsFqdKyYFQQdbexco7EMJQUwiUKcDezhw8Ajj0gxC7/S1aJF8ZuDFAoTQj2L9PcDF15oehTp09bmZDU6asJTkeU+weWyfK4LLgDOPBP4zW9oViV60bGB9bvnADInv/SleB3L2tqyX7f5/POBf/u3fEVLHz4sgZCOQZ/wVAwOxn/obaNQABYvlgeVgpYkhe60Gb8gw9/9nWjIUent1ZPmZDN5TVly8LulOXoqHnooWwK4UgG+/nWpsrNqFQUwSQ6dfZiBiTZ2cZo7VCrROnO5Rl6bQzz6qARMOgSFcCuGh4Gf/MT0KPTg9/l1KcCqVhNLxIoVEqCzYoX87NhDllt092FWSjaPp58erWd03M5crrF2rbic8sThw8Btt5keRTjMZkhZTJZy7opFOxsjTMaWLUrNmdO60cNU+aTEPN3d+udyfdnUoHn7nud8kf/IRGkO4fpRqUhjnVrN9N0PBEwPwFp6erJTlOOyy0zfzWCMjCg1f36wz5TnhdUVrrwymflcX5Ahar3lPJHHIkN+QRgHNusMzGpG1oIaXEhgHx4Grr8+fEcq10zseWJwUEyiuvNVm81nFk5pzfbtEx2XPC8/KZcO1DmgEG6GzibuNhC39WDSxG3e7nIbsyxTqwGnnKJfCNs+n22mcbMyPg489ZT4UrOMxZt1BmY1Y2QkOwK4XLY7gd0v5hBVAAMTUbPELqZPl/KQOrF9PtuOX+Xt/vvFmvDEE8C990aPNncFP9d8+3bTIzkGCuF6qlXJM4uTg2gbStldmGD9+vjlQJWSxYRR0/Zx5516z2f7fHaRSy6RClvFoumRJIulm3UKYZ/hYSmPt2lTdmqu2t560K9GpgO/jRmxi8WLpe+2Dmyfz67hKx0zZwJbtmS/Naulm3UKYeDoAvNZcpGXy3YXJtApNMfGxM9F7OO++4COjvjnsX0+u0S90rF/f3bcb1Nh4WadQjirHZJcKEygu5jD22/rOxfRR1cXcPfd0hAkKi7MZ1fIqtIRBAs362zgkLUOSQ6E5P8f77yj93wOtjHLDf5cDNpT2Mel+ewCWVU6wmDZZj3fmnCWOiS52Hrw+OP1nYtRs/bT3y9zs6dH5mpn5+Sv7ex0bz67QNaUjihYtlnPtyZsmW8gFO3twNlnSxqIq4UJ5s2Tjic6TNKMmnWDhQvlO6/PV92zB/jVr+TvJ5wAnHyym/PZdnQqHYWCbJRcE+gW9hzOtxDW7ZNMk7/9W8n3c5mVK6UoSlwYNesefr4qSQ+dSkd7O7B6tQj1atUda6KFRUnybY7W7ZNMi1ZmPJfQ1W6NUbOETI1OpaNQAN56C3jhBXcEsM+6dVYV7ci3ENbpk0yTAwesi/CLTNx2a6USo2YJCYJOpWNsTNKbXDNHA9YV7ci3EH73XdMjiI5lEX6R6eqK1hsWkNzTu+5i0A4hQdCtdLzxhntaMGBd0Y78CuHhYWDrVtOjiI5lEX6x6O+fEMRBTdMLFgD/+Z8UwIQEZd68eLnaWcKioh35FMJ+rpyLphQgm+k4jekrjSbqQkGOuXOBbduA556jCZqQMOjMHigW3S7va1HRjny2MuztFX+Gqx+9VAJefTW70cDsDUtIMuha+woFt4UwYE2f9fwJ4VpNCpa7mprkeaItPvyw6ZEQQlyjWpWa0XEqZnke8Ad/ALz2mrZhGWHRIuDHPzY9ihyaoy3xA0SG6TiEkKj4gZBxfMPlMnDxxe77l3/6U4kNMkz+hLDLBTpYxJ4QEpf+fskqiNLZyl+D/vqv9Y8rbQ4fltggwznD+RPCLhbo8LyJyc9oYEJIXPr7JbtgwYJgr29cg3QV2jGNBTnD+fMJr1gBPPCA6VEEp1AQ08/atclqwLWamOpHR2WjcvzxktJw1VXpBEOZvj4heeWpp4AbbgB275af6wOuymUJ4lq69Ng1SId/2QYMB7rmTwhfdx3w9a+bHkUw2tqAL3wBuOWW5K5RrUoZt61b5WGrfwB9n8+SJfIAdnXpu64vdL//fWDHDvnZ84Dx8YnX+AtAEtcnhBxNlKyE+t7ErlIuyxprqJZ5/oTwsmVSLcUFkt6hrVkj5qUgqQa6zOHVqph/nnxSBO6hQ1O/hz1lCbEXXxCH6RNtG319wP33G7l0vnzCtRrw9NOmRxGMpDsDXXEFMDgYPNdv3z550OJEEw4Pi/lq0yYJjgsigAF5sHVcnxCin/pCO8Wi6dFE4xe/MHbpfGnCg4PSOs+F6OhKRSZ2En7gNWvkXkQh6rh0ma2SvC/EfRhbYJbdu4H5861sGdiSGTPE6mgClSeuvFIp0avsPioVpYaGkrkHIyNKFYvxxtfbG/6alYqee+N54a9Pss+WLUrNmaNUoTD5vJkzR6mtW02PNPv09Mj9Nr2Ohjna2pSq1YzcrnyZo21PT0ojFenmm48OfopC2A4k69frq9OtlFUdUIhhqlVJs7noImDXrsndK0rJ3xcvltdXq+mOM0/EbU9qgrY2Y4Wc8iWEOztNj2ByCgXxqTzzTHICuFaTdAQdBJ2wtZoEYen0eljUAYUYZHgY+NjHgJ07w71v5055H+MLkqGrC7j2WtOjCMf+/cYaOrQZuaopbA6jv+QS4DvfSfYaGzboEYZhJmwSwtKiDijEEMPDktt68GC09x88KO8HGHGfBC+9JJtll0KODPVoz5cmHKVxfBqUy+nkwI6OxjdF+wSdsEmVCTX0wBAL8FuRxp1X+/dbUbYwcyRh/UqDN94wctl8CeE9e0yPoDlK6e31ORk6feLTpqV/zSjXJ9lDZ4yBBWULM4errqLRUSMbsvwI4eFh4Ec/Mj2KY0k6H7ie44/Xc55iUSrppHnNegqF4Ncn2UK3lsVAP/242iRnfNzIhiwfQtjPUdVlitVJmq0J583T134sqOau85pRrk+yRRJaFgP99GJ7FkorDGzIsi+Eff+RjUFZxWK6rQl1Ca6LLgquuSchLGfPZuGFvJKElsVAP70kYf1KCwMbsuwLYZ3+I92cc066kZk62o8VCtLwIc1rNnLXXfrORdwiKS2LgX76SMr6lQYGNmTZFsI2R+l1dgKXXpr+deMm0q9aFV5z15m8v2AB8IlP6DkXcY+ktCwG+unDdVdRyhuybAthm/08nmdmsnZ1iQk8SrrW5ZcDt9+e7jXr6egA7rsv3jmI2yShZZXLDPTTSRLWrzRJudpXtoWwzVF6F1xgzq/Z3z8hFIM8KMWiNH3YuFHPNaNQKgF3383GDXkniY1rWimCecLF0pU+776b6uWyLYRtjtI780yz169vP1YqHatdFItST3XZMuDZZ6NpwJNds7dXzh0Ev572XXexshHRr2UlmSJYq0m3shUrgOXL5d/BwXykQ+myfpnguONSvVy2WxkuWQJs2WJ6FM0x2ET6GPbuFdP988+LP2TaNDHPrVyZnLa+dy9w223SW/iNN2QxrE8hK5dFQ1m6VHbV1ID1kIVWf9Wq9KXWkfGQRGvMalUCQp98Un6ut8b583rJEpnXaVTKM4mfHjo2ZmdsTjO6u4HHHkvtctkWwnPnSucUG0n5i7YaE5uAvPHP/wz8+Z8D//M/k7/m9NOBb34TuPDC1IYVGR39qZPoWBZG6JTLwJ13Zt/Cs327bEo2bZq8y5VNXHZZ8nX868iuEK7VgNNOAw4cMD2S5ixdCjz+uOlRkKzz7W8DV18NHDoU/D3TpwObN9uvpcXRspISwDfcEC4OpVCQzzAwoG8cNmL7elzPnDnACy+kdrns+oQ3bLA7Ou/VV02PgGSdK64Qa0IYAQzIgvlHf2R/q7/6uIaODhForSgU5HW9vfpbhlarwGc/Gz4Q9MgR8RN//vP6xmIjtq/H9bz4YqpzP7ua8IoVwAMPmB7F5HR2Aq+9RnMrSYYrrgAefDDeOTwP+MY33DCX+i6NalW0mHffFQ25XJZAm7lzRbNPysWxYEH4vsb1FIvA177mxr2Ogu3rcSNJxApMQnaF8PLlYlKzlXIZuOUWKX5BiE6+/W19KTft7cAPf8jAuFZs3SqlXONSLgM/+EE277Xt63EjnicWlocfTvxS2TVH216/lPVqSVJcf72+cx06xFZ/U3HjjXrOs39/du+17etxIyl218quEJ43T/w/NsN6tUQ3u3bpz4/fvDkfua1RqNXEh6iDLLdVdLGedErNHLIrhFeuBA4fNj2K1rBeLdHNZz6j/5wHD9pdAtYkuu9LVtsquliRLCVrZXaFsO20t7NeLdHP9u3JnJeuk+aMjurNfc2qm8rVetIpWCuzK4Q3bAheGtEEhw65U0GGuMPBg8mcl66T5iRRGjer99rFetIpWCuzK4RHR5NbkHRxyy3JaS6E6ISuk+YkEXCU1Xvd1QV88pOmRxGclLprZVcI29y8wWdsLLvRkMQMnZ3JnJeuk+boDjjKclvFahV49FHTowhOSt21spsnbHPzhnpKJame5WLRjiw0A8gaf/InwL//u/7z1mr8TptRqwEzZ+prmeryejAVvb1SP9oVkbNsWSq5zdnVhF97zfQIguFiNGS1Kg/UzJnAzTdLJZzNm+XfdeukRmxvr7yOpEsS5fYWLsymUNCBzoCjJNsqmqZWk65SrghgILV2s9kUwrUa8POfmx5FMFyLhhweljZymzbJ7r9RAxgbk99t2gR8/OP21x/OGnPm6PdT8jtsja6Ao3JZzpVFXFM0AOA3v0nlMtkUwi4VCwfciYb0u8Ts2zf1jlYpEcjXXgusWZPO+Ihwzz36zjVvXjbLKOpERwN7v6tTVu/16Kg+k31apLQuZ1MIu/aFuxANGbdLzJ/+aTLjIsfy6U8Dl18e/zzlcrymBHmiv39CEIdRADwvmbaKtuFCoGwjKa3L2RTCLn3hrkRDXn11vJSvf/on6exD0mHjxniC+Pd+TyweJDj1rRVLpdaR6p2d8pqeHv1tFW3EtdrRbW2prcsWV7OIgUtfeEph8LHYulWPRvTggxLMdfvt8c9FpmbjRgka6u8PJ1Avvhh45JHkxpVlFi6Uzjt+a8XnnwfefBP49a/l7+97H3DSSbLAJ9VW0UbmzZP74oqF8siR1NblbKYoDQ4CX/gCMD5ueiStSbFdVizmzpXGADooFIAf/zi7vi9befFFoK8P+OlPm/vzOzqAf/gH4Mor0x8bcR8/XXFkRNaK/fvFyjd7NrBoEdDdDZxzjjtC+OSTgV/+MpVLZVMI12pyE23/aCk2jo5MrQaccore+rgp5d8RQhKmWpWCQ48/Lg1zmq0ThYKYd084QawCtq/LgFiQnngilUtl0yc8fbr9bQxdiYZMIrVgy5ZstmsjJE/46YqPPCLxIpNt1I8ckb//8pduCGBAtPeUyKYQrtWkQYKNuBYNqbtLjI+LeYOEEGF4GLjppuwG76W4WchmYNaGDWICSUJ4xMH3Aa9da78G7JNEpPn4ePMCJfVlMGs14Fe/kofhxBPFusGSmISYp1rVI4A9z17N+Be/SO1S2RTCo6Pin7CND37Q/iCsRpKKNK9PhPf9Sk8+OWG6asZDD0mZzCVLZCPT1ZXM2Aghk7N+vRTiicsppwBvvSX/ty1gK8UCStk0R9uaJ/zSSxKl6hK6u8T4+InwjWUwW+UiHzgwURLzvPNYTpGQtNFZA/qttyRa/9ZbgRkSkORdAAAd30lEQVQz4p9PJyn2Pc6mELY1T3h8XEL2584Ftm0zPZpgJJErVypJnmS9XynMQ62UvOemmyiISTap1STVcsUKYPly+Xdw0HxAo85YDs+TLIlVq8RCGKfsp27efTe1S2VTCCelveli1y5g8WJgwQL7Ow3p7BJTz/z58f1KviDevl3fuAhpRlpC0fYOZTpLAtc3r9FRf1snxx2X3rVUFtmzR6lSSSnRmew+OjqUGhoyfcdaMzKiVKWi7zP39irV06OU58U/l+fJ+QhJgpERmaul0rFrSrksv+vpkdfFZWhInrOpngvPk9eZWDe6u/Wuf93dR59/9Wrza3KzcSVINgOzpk8H/t//Sy3ZOhYHD0pnIsDelCV/l6ojIrJSAa65BvjUp2S6x0Up+Z737mXUdBDqKxvt3i3aSKkkLRAXLWL0eT1TpeH4wUmbNklp1zhph2FSfpSasAIBU1+zPuvgnXfEXRc100C3q6+xScJLL9kRNZ1mU53UxH3aXHut+d1UmKNSUapaNX3XWhN0p97qMw4NKTUwoNdSUS4rNTho+u7Yja/RdXQoVShMfi89T6k5c5TautX0iM0yNKRUZ2e0+R2WOJamVutGElq8zme38bm1xYKZ8nqC1K6UNldeaf7LDHO4YlatVmWcpZJSbW3RFqgkvpu+PrP3xWaibp7mz9djZnWNkRGl2tujzcMom+mennhzv9m6kZRpW6egLBSUuvBCWQ8GBpT68pdlk2h6LS6VlKrVwn2HMUBqV0ob3b6LDH75sajVZLe4ZIlS06dPPNCNn6dUkkWifmFK4rtJ0YczKXv2yGJyySVKnXWWUqefrtSsWfLzwICZ73ZgILpAAdyIWdDN/PnR71fYzfSePfEFT0fH0XPLF8BhzhFGEOuK56g/ymX954x6pKwMIdWrpYlrmrA/EV01q/pCua9PBGJfn/zcTPBkTRMeGZHNSLHYeoyFQjqm3j17xB3jb47iHnkSxFu2xL9fYTbTAwOt3QNBjkJhYt1IyrRdj+5ATZsOz0vdLYhUr5Ymuv2OaR15MKvq/m7qF6G0Wb16auHb7EjC1Ov7AMO4CYIeHR32xyzoYM6c+PcqzGb6kkv0fD+XXSbni6OlhtHio2jbrhwUwpqwxckf9rDBrJo0ur+bQsGMqffyy+ONu1jUp2GmsSguWKBnrLayZ098rdQ/gm6mzzpLz/VmzUrGtN2KrArilM3R2SzWASRXZCJp0gyNN4Xu72b27PTTatasAR58MN45xseB666LX/UrrY42O3YATz2V7DVMorMaVNDaw7oKX+zfL+OPWzP/8OHg96G/X/qh9/ZK69hCRsSJn/KYEhm5a5Owdm2qNUBjUy5LOcc8sHatvqpmd92l5zxBqVYlJ1QH4+PA9ddHr/qlq6NNUPyc9iyis21n0M20rvWpVJLc77jjP3IkXDWuhQul5OTrrwO33w5cdhkwaxZwxhnABz7gpmD2vFRbrTp4h0LgF5lwBaWSqdVsI11dwJ13AsVivPMsWAB84hN6xhSUdev0tsk8dAhYvTrae9evT7en6+7d5usXJ4Wuxi+FQvDN9OzZeq45d658Nzp44YXw7znxRKkB/Z3vyDhefhn4sz8TDdk16stppkC2hTAAXHKJ6REE59xz81WtqL8fuPHG6O/v6ADuu0/feIJQq0l1JN0880x44eZ3tEmbFLWEVNFZDSroZnrRovjaYqEgm1od7QUBfSZynXWm04atDDWyYUN8bSstnn02f12BBgbEvxp2ISqVgLvvFnNYmmzYIBYL3Rw5Anz963aMpRVHjqSqJaSKrsYvYWIUVq4E2mJWD25rk/Pocu/oOo+tLWWDkGJsTvaF8Oio+N1cYP/+fLbnu/12EUBB/GOeJ/Wn77rLTK1tnX7DRjZvDj+WAweSGUsrUtQSUkWXKyhMjML06cCyZdGDFD0P6O4WoT9nTrRzNDJ3rp7z2NpSdipSjs3JvhB2bTeW1/Z8/f3AD34gkZal0rECuVyW3/f0iOnWVLOLJOfTnj3hXm9qbmc1gl9H1H6UGIU4AaTlsrwf0Gva1oHtLWUnI+XYnOwLYRd3Y2NjEnCTN/xIy1dfBW65Bejrk11+X5/8/Oqr8ve0TdD12DSfTIylWMx2BH8cgRg1RiFqL91KRd7nPw86Tds6cDHI1POApUvTjc1JNSvZBK5WznKpjnSeGBgI310n6PHhD4cfi67iEkEPU4VR0iRKEYpSKX7hFR1NF5JoBhGHJOpMJ3kY6GaHVK9mAlcrZ6VRR9pvOHDllVKpy+9mkvVFNg5Jzqebbw4/liglM+McM2YkclutI6kuRFNR36WsXD52TWjWEKWeNGpHh2HDhvQ3ilEPnd9jCJD6FU3g2m7MP5KqI51En9E8EVfbaHZE1TCXLk13Tl53nf77aStxBWIcwjREaSTpLkpBidt/PM2jUDDWpMRTSqn0jN+GqFaB885Lt6iBDrq7gcce03tOv8Th2JhMv8nwPPGN3XGHuSAoW0liPp1/PvD009HGcu65yUVs19PWBvz3f+crlx2Q/O0NGyQ16+23JTDtQx8Sn6et98L0c55WKVVdnHQS8OabZq5tRPSbYGjIHbOIf+jWhG3ZIWcBncXr43YoWr06nfmYcmF7EhNTmryLrQ6LRWNuuHxowj6LFoWri2qSclkiglet0nO+ONpbpSJpQSajkm1Ex26/o0OKjsTVQq64In5DiVbkeQ7UaqIJj45KWtjxx0v6zVVX2asJ15O2Jt/bC2zaJOLNFUol4NZb9a23YTAi+k0xMBC/1Vdah+7oaNuiJrOCr21E6eGrI6K2njVrkgnUyqs1ZGREqSVL5LtttKIVCnKvly5l7EQ9rgbCAsZ6ucPIVU3h0gTRKfTS7jOaR2o1CVqaMaN1IIrnyb1MKqCnWlVq2TIRHI0C2Q/EW7AgWJqV7shfl1i9OviGplCQ1xN3U0IBY73cY2Z2O4ZfEeeRR0yPpDWVykQVHB3o7DNqwlzjAieeCNx7rxy++a9alY40+/eLuWvuXCnMkGRAz8KFUv5yKhPk9u1SEOaJJyQ4p774f7ksy9LSpTIP82aCDmvaP3IEGByUYjIbNyY3LhdwuWmDoUpw+RLCAHDNNXYL4WLx6Co4OjDRZzTP+G3dbB6DX53MxcjfJFmzJrpv/cEHgZkzpRZ6XnGtTLCPwV7u+RPCO3dKqkVczTApxseBf/xH4Le/1Rf4sWtX/HMA0fqMEruxYcNgC9Vq/P7jd9wBXHpp/qwHPjaVdQ2DUsbKbGa/dnQjo6P2CmCfahX48peB006TSMO4Gqgu85CrZibiLrWamHpXrACWL5d/BwfD914Owrp18S1G4+NynrwybVr8JhJpY6JedP3llVLKyJVNsXx5+JZxJtGRTD97NvDii/HHMmsWsHt3/PMQMhXVqvisn3xSfq7fAPo+6yVLxGeto+tPrQa8//162p7mtajJ8DBw441Hxxe4gOH0O8e2LBpwzVyi1ER7w6h9hmfP1jMWXX1GCWnF8LDktG/aJMK30QIzNia/e+QR4OMf19N/e8MGedZ0oJScL0/UV+hyicZOVAbInxB2tcdlnD7DtvUZJWQy6gugBBGKY2PAtddKQFUcRkf1lf4cH5dAt7xQrbpVotLHF8CGy/LmTwi72OPSJ2qfYdv6jBLSjKiLuZ8i9PnPR7+27qjet9/Wez6bWb/ePQ142TIxQVtQFz9/QtjPFXYRpSSvM2xQyvTpMuk8L9p1PU+aSeTNx0XSZf36eNrUHXdEN03rdlMZyjlNnVpN/PYuhRadeqrEBVkSwZ4/IQxIMIdrEXw+nhfN37R2rQS0RKFc1ls8hJBGajXg8cfjnWN8XAKDorhs5s0DOjvjXd+nVDKWc5o6Gzak08FLJ2ecYXoER+GoJIpJVxdwzjmmRxGNsbFo/qauLtEUKpVw77MgcIHkAB1V3QAJ2IrqsolqKZrsfHngoYeAgwdNjyIcFMKWcOmlQHu76VFEI6q/qb9/QhBPteB4njWBCyQH6KjqBsRz2ehwUxnOOU2V4WHgJz8xPYpwGKyMNRn5FcIrV0qJSBeJalYGRKA+8wzQ0yNms8Zzlcvy+54eawIXSA7QVdUNiOeyCWspaiQvrhs/iM41U7TByliTkb+ylT6uNHNoxsMPSyWtqIUKWDeY2IbOamxxXTZR022y6LqZrJfyM8+4FxENWGmlyF/FrHqqVeAjH9FTJSdtmlXScr35OMkvuqq6+XR3A489Fu299bnKQcma66ZVxbJSyc0StsUi8Oyz1m2S8i2EAdH8XG5MUKlIsYKXXkqvxB8hurn0UrHO6KKvD7j//ujv91s9bt4sAWPNzK6FguTPd3dnq+VjffWrLImHuXOtLKKSX3O0z/z5bgvhffukUMFk+CajTZuArVuztVsn2WHRInEN6fAx6gi+aXTZmOgNbYIoVgBXOPts0yNoCjXhwUHg5pvdNK9EIWtmM5INajVgxgw96S6lEvDqq9kSjmlQrUrN7iwK4HIZuOUWK9t2UgjXatKIOy9CGDDeNYRMge/bHxmRqOH9+2URmT1bNMas+vh7e8ViE2dJ8jyJ7Ndp2s4LOu6/rVi8MaMQBrI9+ZrBhao5pgPb/GCYxx+f2g+5bFn2fPw6NLG8bzCjzuEsKyO2r3eKKDUyolSlopSI4XwcpZJStZrpO28HIyNK9fTIPSmVjr5PhYIcc+YotXVrcmMYGgo3Bz1PXj80lNyYTBD2PtQfWbwfQYk7hwcGjn1fVo5KRalqNd3vIwQwPQBriPPwu3iUy0oNDpq+6+bxv3fPC3bf5s+XBS+JMURdYLImeLghCYeOOXzllebXpCQOB+YFTA/AKoaGlGpvNz9x0jr6+kzfcbNEFX7t7foebB1WGMt3+pGoVpXq7VWqo0O0uGafu1CQv/f2Zu/zByXqHO7oOHoOd3ebX490Hg5tzOgTbmTbNsmpda0cWxTiFDRwnbj+x85O4KtfjR9lzmCk1uQtRSgMcedwqQTcdZfM4RUrgAce0Do8oyxbBqxb50RsAIVwM3p73SxnGZa4BQ1cRofwixsEpDMYxuLoT2sxHYgXF51z+Omns5OqWSoBt95qZTpSM/LbwKEVOgq5206hYF03kdTQ1Yh8375obfN8ojQZmIyoTQvyyNatokmfcoo86w88IJWxHnhAtKfTThMBV62aHunk6JrDY2Myh1eu1NNK0gb277eyMtZkUAg3I2rvXdewrJtIaugUVlHa5vmMjurTPKI2LcgT1SqwYAFw0UWSf33kyLFup7Ex+U42bRJT7/CwkaFOia45rJTM4W9/O1suuKjtXg1AITwZfu/dOG0DbWb2bDdMbkmgU/jF0UDfeUfPGHwcWnhSZ3gY+OhHgZ07g71eKbF03HSTnYJY5xxWCvjiF7MlhKdNMz2CwFAIt6K/H/jBDyQIJ2tMny4lO6NqcS6jU/jF0UCPP17fOACnFp5UGR4GPvc54NCh8O/1BfH27frHFQedc/jAgWj3xlZ01A5PEQrhqVi4EDj/fNOj0M/TT7vj/9KNbuEXVQOdN0+CSHTg2MKTGn7z+QMHop/D95vahO45nCWUcsrVRiEchPPPB9rbTY9CP674v3SjU/gB0TVQnQuFYwtPaqxfH78hge83tclqpHsOZ4mlS51ytVEIB2HlSmkInVV8/9f110uDgOXLJW8wq+ZqncIqjgY6fbrkpHtevDF4nnMLTyr4EcQ6sC36nBuu5hQKEvHuEBTCQdC1WNrOoUNivnMtXSMsOr/PuBro2rXxg//KZecWnlTQKTRtiz735zA5moULnSjQUQ+FcFB0LJaukWVztY7vU4cGGjcdzu8P7djCkwo6I4gB+6LP81DPIAzt7cCll5oeRWgohIOSl9zhZtierhEF//uM41fTpYH66XBh5pbnTQjguKUzs4ruFDDbos/9OZzF7I0oFItOmukphMPgL5ZZ9g+3wtZ0jaj090vt3I6O8O/VrYH290v5wN5eGU9hkkezUJC/9/TI6ymAJ0dnBLGt0ef9/VLDPIuBo2FwOC6CtaOj8Fd/BXzjG9lKbg9KFpsFbN8OXH01sGPH1K/1PFmQk9RA2bRAD4OD+uohm6rNHbS+dZg5nEXi1nE3CIVwFHQW3neRrDYLeOop4IYbgN275ef6TVaxKGb5U08FLr4Y+NKXsvf5s4au59TExrNalfQqP7q7/jOUyzIXlywRd8jMmROC+qWXgP/6L/2meJtx3C1DIRwVHR1MXKVQAP7yL4F77zU9kmTYuxe47Tb5ft94Qxbh8fGJv/um4lmzxJx94YVmxkmmxoZuWWEZHha3z9hY63F7nsxFzwPa2o4W1J6Xj7WpXAbuvNNZAQxQCEcnbi9P1ykWga99zenJPylBF0Gf+fOB++4TUzGxC509d9PAn3t5XVfC8P73A9/9rpMm6HoohOOQ9wfGcTNQU6J+p+3twD332H8vfB/jyIh0Etq/X7SJ2bOlUIsrvXTDEPU77egA7r47ve807xv7MJTLUtffcQEMAFAkHkNDSlUqSnmeUqI35euoVJSqVk1/C3oYGZHPE/VedHbKfLCRkRGlenqU6uhQqlBoPv5CQf7e0yOvzxJhn9MFC9Kf1z09+V1HwhzFor3PWQRgegCZoFpVqrdXFjDTE9TEcd55pr8BPehYBG3clPgCKOhn8Dx5fYYWOqXUxHNaKilVLh/9mQsFOebOVWrbtvTHtmePjMv0s+zC0dOT/veTIDRH62TvXuCaa4BHHz06kCcPDA4Cn/50sHQKG9EZ8d7ba08KVxyXSRbdDcBECtjzz0sVrGnTJAfYZOqXznSqLNPRAbz+uv3rSQgohJNgeFhSXfL0QHmeBGs1Rmk2plPYGryUhZzSRnT4GB3Ov3SKFSukXjuZnCzWKAArZiVDf78kzucJpYDDh48VYq7Un9ZZZ9iWjjvr18v9j4ONvXSzSJ7yeqOS0UYlbaYHkFneesv0COxCqYmyl4B9Jk6di6ANHXf8Nn5xDV1KTfTSNa3Zu8xUla90ltjMIsViZhuVUAgnBXe2zfEFcVeXXQ+U7kXQdMcdnZq4r9mvWqXvnHmhVeWrjRtFs5s1Czj3XHFj5MmFFYZPftK+jbsmKISTgjvbydm3D1i+HLjggskDt4LWzNXFvHnia9K1CJruuKPTvG6DZu8iUxV98cui7tolB2lORwfwzW+aHkVi0CecFPPmxWuTl3XefFMCUdatA047TSKKq1U5enslUvnmm+U1mzc3f61OdLZAs6Hjjm5LjGnN3jXqo9IZ+xodzwO6uzPtCmF0dFLkvclDWPz6t54HHDrUeuFKqpORrnrgNkRH64627esD7r9f3/myDCtf6SMH0fnUhJNi+nRJy/E80yNxA6VE+B48OLUQrA/y0hltvXatCPc42NLXVKclxgbN3iV0RKUT/T27LYWacJJwR5w8unfKcXO8bdm567TE2KDZuwItYPFJo2e3RVATTpKuLplIlYrpkWQX3Xms/f3SNaejI/x7bdq567LE2KLZu4IN+eGuUi7Lhq+nRzayORDAADXhdAjbGo+EIwlNbft2KbiyY8fUr7V1586KWenhR/MPDwOvvGJ6NO5xxhny7JgsHWoIasJp0N8vC1lPjwiMAm+7VpKoULVwIfDcc8C2bcDcufKdNX5vtu/c41pibNLs06ZWk1KmK1ZIOt2KFfLz3r1Hv64xmp8CODzlsjw7q1blTgAD1ITTZ+9eYPFiWeCJPpKO3rWx6H9QwjZxsFWzT4NWxTUa66Bv304Llw5yHnNAIWwCdkzRT3c38NhjpkdhL9u3i3DZvFlqfPuFIuopFCRNrLtbhEzeNOCgbiM/nQ6QiH4SnYw2ZQgDhbAJGEGpn1NPFf+nK60TTeFr9NUq8MILMgdLJTG5d3W5odknQZyWjyQ6jDmgEDaGrsIQ5GhcaZ3YSNplOskETCU0Q1b7VYeEQtgUfPCTxRW/ZisfZLEon+PCC6VkpysbCtfghjhdXHk2U4JC2CQ0gSVPkN22r4WOjAC7d4tPsFQC5swBFi1KThsNk7pWLAI33ggMDOgfRx6p/84feaS5j5zoxbdSLV2az5iDSaAQNg1ziJOnWAQ+8hHxw9ebeH0t9PHHpw5WWrbsWPN2HBNy1A3Y5ZdLCzwSjVaWB5IMJ58MfOIT7mQTpAyFsA34katPPCGmGtadTQ5/N/7BDwI/+1mwWtU+9XmzQdNYmpmQ47oi1qwBbr892nvzRv1GaedO4MUXZbPFZS8dGHg1JRTCNtGYi1qrAT/9qWhpxA7a2+N3eorrgywUgB//mAtbK6jxmoeBV4GgELYdmqvdpnEh0pWetmyZ5PzagMnI7mbXfvddYOtWucd8ZtKHgVfhUMR+qlWlenuV6uxUSpYVHi4dlYp8h0opNTCgVKkU/5zFolK1mtl5OTKiVE+PfJ7Gz1Quy+96euR1SV6bz4Udh/+d9/ZOzHcyJdSEXWLvXuD884Fdu0yPhITBrwo0PAxcdJGekqXFophbV60K9nrd2mqY6lK+VnTJJXrGwKwCeygWgVmzgLPPZuBVVEzvAkhIRkZEszK96+UR7igURGMrFPSds68v2HzRra0ODYWfg4WCUm1t8ccQ5do89B+eJ9/D0FDweUOaQk3YRagJEGDqetlRtNWpfHhJFZnxPMnNXrwYOO645pry1q3ymRmoaI62NjmY66sNCmFXYcAWadU5KspGLUg067JlkkqXFqWSpBSdcALw5puc6yaZMQO47jqanDVDIewyzC/OL54nBUjOOONYrfGVV6Jrq5PldVar0vnLT/kh+aKjA3j9dQrfBKAQzgJ+fvH3vy8mO5bgyyd+oZBp0+JpjSefDFxwwYRQf+ghWl3yDNsNJgqFcNagv5joolyeKEoyPm56NMQUrHqVKG2mB0A04/vzqLmQuNC9QepLtZJEoCacVegvJoREhVWvUoNCOOs01qOeNg14z3uAl18Gvvc9CmhC8oqfFlb//LPdYOpQCOcZX0CPjAD/8i8M6CIkL5RKwKOPAjt2HL1BZ9Wr1KEQJkLczj6EEDdgdyOroBAmQlKVkAghdkA/r5UUTA+AWEJXlzyclYrpkRBC4lAqHf1zuSy/6+mRVCMKYKtgihKZgOlNhLiL50lA1cc/Tj+vQ9AcTY6F6U2EuAeLajgJhTCZnGbpTb/7HbBlCwUzITbBYCtnoRAm4WEHJ0LsgMFWzkMhTKLRymTd3i41hwkhU9PZCRw4MPGzXzDjIx+Rn3/0o2OfMRbVyAwUwiQezUzWH/qQLBC33BJNWy6XpXDIe98L7NnDIiIkm1QqwLp18v9WgVSTPWMMtsoEFMIkOVppy/5O/oILgDPPBH7zm+YLDE3fJGvQhEzqoBAmyRN3J18vzA8epGZM7KZQkKO9nSZkMiUUwsQd9u4FbrsNGBpif1tiF40CduZMmpBJICiEiXv4JmqW2CSm8DzRdmfNAs4+mwKWRIYVs4h7hKns5S+W06cDtRo1aBIPmpSJZqgJE3cJEvhVv1gyyIsEpa0NKBaBD3wAmDFD/k+TMkkACmHiPmECv1oJ7kJBgr78f0n+KBaBc84BLr2UwpakAoUwySeTCe7ubmDzZvn9K6/I8frr1Jxdp1gEfv/3gXfekc1XvVuCJmZiEAphQqZi717gmmuARx+lT9k1GnNyWfiCWAaFMCFBmcoHPT4OvO99wK9+JZpX/d+LRTFxF4vA4cPpjz3LFApARwewf//E76jdEkegECYkLFNpU1P9/StfAb74RdbXDkqx2Np8zJxc4jAUwoSYgD2bJ2hrO9o6UC9kP/MZYMcOCliSWSiECTFJM635Pe8BXn4Z+N73gkdwt7XJz+3tUtqz0ezt//2444B33xUhZ+rRp5Al5P+gECbEVoJEcEcxhwfNr/7Up4DvfjeaDzxogw5Ccg6FMCF5JWikcFwfOCFkUiiECSGEEEMUTA+AEEIIySsUwoQQQoghKIQJIYQQQ1AIE0IIIYagECaEEEIMQSFMCCGEGIJCmBBCCDEEhTAhhBBiCAphQgghxBAUwoQQQoghKIQJIYQQQ1AIE0IIIYagECaEEEIMQSFMCCGEGIJCmBBCCDEEhTAhhBBiCAphQgghxBAUwoQQQoghKIQJIYQQQ1AIE0IIIYagECaEEEIMQSFMCCGEGIJCmBBCCDEEhTAhhBBiCAphQgghxBAUwoQQQoghKIQJIYQQQ1AIE0IIIYagECaEEEIMQSFMCCGEGOL/A0yR0xX7TK6YAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "nx.draw(G)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 安裝Louvain\n",
    "- pip install python-louvain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: python-louvain in /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages\n",
      "Requirement already satisfied: networkx in /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages (from python-louvain)\n",
      "Requirement already satisfied: decorator>=4.1.0 in /Users/davidchiu/.pyenv/versions/3.6.2/lib/python3.6/site-packages (from networkx->python-louvain)\n",
      "\u001b[33mYou are using pip version 9.0.1, however version 18.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "! pip install python-louvain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "import community\n",
    "comm = community.best_partition(G)\n",
    "#comm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "cluster_ary = np.array(list(comm.values()) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles_ary = np.array(titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "【狗仔偷拍】小嫻搬離何守正家租66坪房　月租6萬元\n",
      "許聖梅心疼小嫻被當空氣　爆何守正「有兩個女學員」\n",
      "【動畫解盤】毒菇跳火線譙seafood　小嫻難瘦香菇\n",
      "【獨家】小嫻賣房求子　婆婆竟拒入籍何家\n",
      "小心！在美結婚台灣沒登記　偷腥照樣能捉姦\n",
      "小嫻離婚導火線　拉何守正信妙禪\n",
      "小嫻別傻傻被欺負！女律師說「姐寶」就要這樣對付\n",
      "小嫻守正結婚在台沒登記　想離婚只有兩條路\n",
      "大姑出面護弟！轟小嫻不能生「媽媽是全台最沒有尊嚴的婆婆」\n",
      "小嫻信奉妙禪　關鍵原因與何守正有關！\n",
      "小嫻何守正想離婚　必須先做這件事！\n",
      "「小嫻不快樂！」　許聖梅：何守正虧欠她\n",
      "【內幕動畫】小嫻婚變何守正姊反擊　不滿媽煮飯侍奉星媳婦\n",
      "小嫻多信妙禪？　曾見證「師父帶我跳舞」\n",
      "【話當年】被拍和她上賓館　何守正掰了阿妹\n",
      "小嫻婚變無徵兆　男星嘆：兩人向來出雙入對\n",
      "【獨家內幕】太傷！小嫻被分手　何守正當小三面前攤牌\n",
      "【小嫻離婚】何守正稱沒有遺憾　人妻女星超火「一嘴屁話」\n",
      "【小嫻離婚】3大退讓人財兩失　求子花光430萬積蓄\n"
     ]
    }
   ],
   "source": [
    "for rec in titles_ary[cluster_ary == 67]:\n",
    "    print(rec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(3, 30),\n",
       " (67, 19),\n",
       " (64, 13),\n",
       " (134, 9),\n",
       " (108, 8),\n",
       " (188, 8),\n",
       " (20, 6),\n",
       " (7, 5),\n",
       " (135, 5),\n",
       " (229, 5)]"
      ]
     },
     "execution_count": 248,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "c  = Counter(list(comm.values()))\n",
    "c.most_common(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<899x38653 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 128155 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[6, 0, 0, ..., 0, 0, 0]], dtype=int64)"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = np.sum(X[cluster_ary == 67],axis = 0)\n",
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = np.squeeze(np.asarray(m))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([14456,  5361, 36950, ..., 25530, 25529, 19326])"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n.argsort()[::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_name = np.array(vectorizer.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['小嫻', '何守', '離婚', '結婚', '美國', '婚姻', '時間', '表示', '何家', '台灣'],\n",
       "      dtype='<U18')"
      ]
     },
     "execution_count": 254,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_name[n.argsort()[::-1][0:10]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 文章分類"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "news = pandas.read_excel('https://raw.githubusercontent.com/ywchiu/pytextmining/master/data/20171214news.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "#news.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "生活      160\n",
       "社會      146\n",
       "國際      127\n",
       "財經地產    110\n",
       "娛樂      101\n",
       "政治       84\n",
       "體育       62\n",
       "論壇       38\n",
       "副刊       25\n",
       "3C       22\n",
       "動物       19\n",
       "壹週刊       5\n",
       "Name: category, dtype: int64"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news['category'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = []\n",
    "titles = []\n",
    "tags   = []\n",
    "for rec in news[news['category'].isin(['社會','娛樂'])].iterrows():\n",
    "    tags.append(rec[1]['category'])\n",
    "    titles.append(rec[1]['title'])\n",
    "    corpus.append(' '.join(jieba.cut(rec[1]['content'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer()\n",
    "X = vectorizer.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "train_data, test_data, train_tag, test_tag, train_title, test_title = train_test_split(X, tags, titles, test_size=0.3, random_state=42)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 278,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "\n",
    "clf = MultinomialNB(alpha=0.01)\n",
    "clf.fit(train_data,train_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted = clf.predict(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9866666666666667"
      ]
     },
     "execution_count": 280,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(test_tag == predicted) / len(test_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['娛樂' '社會']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[32,  0],\n",
       "       [ 1, 42]])"
      ]
     },
     "execution_count": 282,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "print(clf.classes_)\n",
    "confusion_matrix(test_tag, predicted)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [],
   "source": [
    "title_ary = np.array(test_title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['【K律師論點】離婚＝失敗？\\u3000K律師這麼說'], dtype='<U36')"
      ]
     },
     "execution_count": 286,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "title_ary[test_tag != predicted]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 正負向情緒判斷"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "movies  = pandas.read_excel('https://raw.githubusercontent.com/ywchiu/pytextmining/master/data/yahoo_movie.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies = movies[movies['status'] != 'soso']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /var/folders/46/b7dzk4mn6g54qzptv608w7d00000gn/T/jieba.cache\n",
      "Loading model cost 0.931 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    }
   ],
   "source": [
    "import jieba\n",
    "jieba.load_userdict('userdict.txt')\n",
    "corpus = []\n",
    "scores = []\n",
    "for movie in movies.iterrows():\n",
    "    corpus.append(' '.join(jieba.cut(movie[1].content)))\n",
    "    scores.append(movie[1].status)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<980x5900 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 16897 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "vectorizer = TfidfVectorizer()\n",
    "X = vectorizer.fit_transform(corpus)\n",
    "X\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "train_X, test_X, train_Y, test_Y = train_test_split(X, scores, test_size = 0.3, random_state = 42)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "clf = MultinomialNB(alpha = 0.01)\n",
    "clf.fit(train_X,train_Y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7653061224489796\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score,confusion_matrix\n",
    "predicted = clf.predict(test_X)\n",
    "\n",
    "print(accuracy_score(test_Y, predicted))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['bad' 'good']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 83,  34],\n",
       "       [ 35, 142]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(clf.classes_)\n",
    "confusion_matrix(test_Y, predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5.473129708653799, '睡著', 7.08742424672882, 0.2949490737488992)\n",
      "(4.792868437938559, '浪費', 6.154256516095439, 0.2840445332028395)\n",
      "(3.6585428519078294, '看到', 13.329343099180784, 2.6433475399174013)\n",
      "(3.540974554929691, '睡覺', 3.766948224245918, 0.06381680122542253)\n",
      "(3.219715305042073, '演什麼', 3.219715305042073, 0.0)\n",
      "(3.200873662775316, '失望', 7.023414431232068, 1.1942179452163755)\n",
      "(2.630248789850424, '爛片', 3.2845012276293253, 0.24874165527744876)\n",
      "(2.5847956452270666, '大爛片', 2.9986121489472772, 0.1600964101298841)\n",
      "(2.4960032136317496, '第一次', 3.1059966596004607, 0.24438808517443952)\n",
      "(2.4928224794774314, '只有', 4.363951639706871, 0.7506066619800713)\n",
      "(2.4821423830971843, '最難', 2.4821423830971843, 0.0)\n",
      "(2.385844296280842, '一顆', 3.057032484414935, 0.2813210355681509)\n",
      "(2.2468160175173875, '空洞', 3.076030980477971, 0.3690622447479354)\n",
      "(2.226675825141232, '第五集', 2.410521805158007, 0.08256522028980767)\n",
      "(2.184939663427423, '超難', 2.184939663427423, 0.0)\n",
      "(2.041075983456699, '難看', 3.723557669139378, 0.8243111473161739)\n",
      "(2.0114327087398904, '有夠', 2.5322797825243093, 0.2589433250842957)\n",
      "(1.9105924945110568, '小時', 2.4241392862201945, 0.26878928561925514)\n",
      "(1.9004041786487231, '爛透', 1.9004041786487231, 0.0)\n",
      "(1.8856994530429616, '七八', 2.3784233248025903, 0.261295017593879)\n"
     ]
    }
   ],
   "source": [
    "import operator\n",
    "coef_features_c1_c2 = []\n",
    "\n",
    "for index, features in enumerate(zip(vectorizer.get_feature_names(), \\\n",
    "                        clf.feature_count_[0], clf.feature_count_[1])):\n",
    "    feat,c1,c2 = features\n",
    "    coef_features_c1_c2.append(tuple([c1/(c2 + 1), feat, c1, c2]))\n",
    "\n",
    "for i in sorted(coef_features_c1_c2, key = operator.itemgetter(0), reverse=True)[0:20]:\n",
    "    print(i)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}