3-gram-language-model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 100.00 % [==================================================>] 10000000/10000000 \t used:40s eta:0 s"
     ]
    }
   ],
   "source": [
    "from utils import ProgressBar\n",
    "from collections import Counter, defaultdict\n",
    "sentlength = 10\n",
    "enlines = []\n",
    "START = '<START>'\n",
    "END = '<END>'\n",
    "\n",
    "pb = ProgressBar(worksum=10000000)\n",
    "pb.startjob()\n",
    "num = 0\n",
    "# 只取英文句子\n",
    "with open('data/segmented_train_seg_by_word.txt') as fhdl:\n",
    "    for line in fhdl:\n",
    "        num += 1\n",
    "        # 使用行号作为标注，奇数行为英文，偶数行为中文\n",
    "        if num % 2 == 1:\n",
    "            enline = line\n",
    "        else:\n",
    "            continue\n",
    "        \n",
    "        enlinesp = [START, START] + [i.lower() for i in enline.strip(\"\\n\").split()] + [END, END]\n",
    "        # 设置一个最大长度提升demo速度\n",
    "        if len(enlinesp) <= sentlength:\n",
    "            enlines.append(enlinesp)\n",
    "        if (num // 2) % 1000 == 0:\n",
    "            pb.complete(1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = defaultdict(lambda: defaultdict(lambda: 0))\n",
    "\n",
    "# 统计3-grams\n",
    "for line in enlines:\n",
    "    for i in range(len(line)-2):\n",
    "        w1 = line[i]\n",
    "        w2 = line[i+1]\n",
    "        w3 = line[i+2]\n",
    "        \n",
    "        model[(w1, w2)][w3] += 1\n",
    "\n",
    "# 把计数转换成概率\n",
    "for prefix in model:\n",
    "    total_count = float(sum(model[prefix].values()))\n",
    "    for word in model[prefix]:\n",
    "        model[prefix][word] /= total_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "\n",
    "def generate_a_sentence(model):\n",
    "    text = ['<START>', '<START>']\n",
    "    sentence_finished = False\n",
    "\n",
    "    while not sentence_finished:\n",
    "        # 设定一个随机阈值 (增加文本多样性)\n",
    "        r = random.random()\n",
    "        accumulator = .0\n",
    "\n",
    "        for word in model[tuple(text[-2:])].keys():\n",
    "            accumulator += model[tuple(text[-2:])][word]\n",
    "            # 当累加概率超过阈值时，选择该词汇\n",
    "            if accumulator >= r:\n",
    "                if word == '<END>':\n",
    "                    sentence_finished = True\n",
    "                else:\n",
    "                    text.append(word)\n",
    "                break\n",
    "\n",
    "    sentence = ' '.join([t for t in text[2:] if t])\n",
    "    return sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "he is my future .\n",
      "will do .\n",
      "reliable ,\n",
      "who ' d be invisible . visible -\n",
      "i get sunburn\n",
      "but that day cannot come over .\n",
      "hhello , come behind me\n",
      "listen , what an accomplishment\n",
      "she ' s so bossy .\n",
      "i feel so sad .\n"
     ]
    }
   ],
   "source": [
    "# 生成10个句子\n",
    "for i in range(10):\n",
    "    sent = generate_a_sentence(model)\n",
    "    print(sent)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}