Q_learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#CODE FOR INITIALIZING REWARD TABLE\n",
    "import pickle\n",
    "from pprint import pprint as pp\n",
    "import datetime\n",
    "import glob\n",
    "import random\n",
    "import os\n",
    "from textblob import TextBlob\n",
    "import pandas as pd\n",
    "import sys\n",
    "\n",
    "\n",
    "data_path = \"data/300_convo/\"\n",
    "\n",
    "\n",
    "rel_path = \"strategies.pkl\"\n",
    "abs_data_path = os.path.join(os.getcwd(), data_path)\n",
    "abs_strategies_path = os.path.join(os.getcwd(), rel_path)\n",
    "\n",
    "\n",
    "N_TURN = 3\n",
    "list1=[]\n",
    "bot_strategies=[]\n",
    "learning_rate=0.9\n",
    "discount_factor=0.9\n",
    "epsilon=0.9\n",
    "\n",
    "user_utterances=[]\n",
    "strategies_list=[]\n",
    "\n",
    "evaluation_dict={}\n",
    "user_lines=[]\n",
    "all_state_variables=['qs+b',\"qs+e\",\"qs-b\",\"qs-e\",\"qs'b\",\"qs'e\",\"q'ls+b\",\"q'ls+e\",\"q'l's+b\",\"q'l's+e\",\"q'l's-b\",\"q'l's-e\",\"q'l's'b\",\"q'l's'e\",\"q'l's'b\",\"q'l's'e\",\"q'l's+b\",\"q'l's+e\",\"q'l's-b\",\"q'l's-e\"]\n",
    "#state action table\n",
    "R_table=pd.DataFrame.from_items([('qs+b', [0.0, 0.0, 0.0,0.0,0.0,0.0]), (\"qs+e\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"qs-b\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"qs-e\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),\n",
    "                                           (\"qs'b\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"qs'e\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"q'ls+b\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"q'ls+e\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),\n",
    "                                            (\"q'ls-b\", [0.0, 0.0, 0.0,0.0,0.0,0.0]), (\"q'ls-e\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"q'ls'b\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"q'ls'e\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),\n",
    "                                           (\"q'l's+b\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"q'l's+e\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"q'l's-b\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"q'l's-e\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),\n",
    "                                          (\"q'l's'b\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"q'l's'e\", [0.0, 0.0, 0.0,0.0,0.0,0.0])],\n",
    "                                            orient='index', \n",
    "                                          columns=['question', 'elaborate', 'opinion','affirmative','negative','neutral'])\n",
    "\n",
    "#temporary dictionary which stores the count of the recorded utterance till date,this is used for calculating the average to be stored in state table\n",
    "temp_dict={\"qs+b\":0,\"qs+e\":0,\"qs-b\":0,\"qs-e\":0,\"qs'b\":0,\"qs'e\":0,\"q'ls+b\":0,\"q'ls+e\":0,\"q'ls-b\":0,\"q'ls-e\":0,\"q'ls'b\":0,\"q'ls'e\":0,\n",
    "           \"q'l's+b\":0,\"q'l's+e\":0,\"q'l's-b\":0,\"q'l's-e\":0,\"q'l's'b\":0,\"q'l's'e\":0}\n",
    "\n",
    "Q_table=pd.DataFrame.from_items([('qs+b', [0.0, 0.0, 0.0,0.0,0.0,0.0]), (\"qs+e\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"qs-b\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"qs-e\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),\n",
    "                                           (\"qs'b\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"qs'e\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"q'ls+b\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"q'ls+e\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),\n",
    "                                            (\"q'ls-b\", [0.0, 0.0, 0.0,0.0,0.0,0.0]), (\"q'ls-e\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"q'ls'b\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"q'ls'e\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),\n",
    "                                           (\"q'l's+b\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"q'l's+e\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"q'l's-b\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"q'l's-e\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),\n",
    "                                          (\"q'l's'b\", [0.0, 0.0, 0.0,0.0,0.0,0.0]),(\"q'l's'e\", [0.0, 0.0, 0.0,0.0,0.0,0.0])],\n",
    "                                            orient='index', \n",
    "                                          columns=['question', 'elaborate', 'opinion','affirmative','negative','neutral'])\n",
    "#_______________________________________________________________________________________________________________________________\n",
    "#utility functions\n",
    "def check_strategy(response):\n",
    "    for key, value in strategies_dict.items():\n",
    "        if response in value:\n",
    "            #print(key)\n",
    "            break\n",
    "            \n",
    "    return key\n",
    "def check_strategies(response):\n",
    "    temp_list=response.split(\":\")\n",
    "    temp_list2=temp_list[0].split('_')\n",
    "    return temp_list2[1]\n",
    "    \n",
    "\n",
    "def get_utterance(line):\n",
    "    temp_list=line.split(\":\")\n",
    "    value=temp_list[3]\n",
    "    return value\n",
    "\n",
    "def max_beginning():\n",
    "    value=''\n",
    "    max=0\n",
    "    for column in Q_table:\n",
    "        for values in all_state_variables:\n",
    "            if \"b\" in values:\n",
    "                if Q_table.at[values,column]>max:\n",
    "                    max=Q_table.at[values,column]\n",
    "                    #print(max)\n",
    "                    value=column\n",
    "    return value\n",
    "def populate_evaluation(line):\n",
    "    list1=line.split(',')\n",
    "    for values in list1:\n",
    "        if values is not '':\n",
    "            list2=values.split('=')\n",
    "            evaluation_dict[list2[0]]=list2[1]\n",
    "\n",
    "#utility functions\n",
    "def next_state(utterance_no):\n",
    "    if not utterance_no==len(user_utterances)-1:\n",
    "        return user_utterances[utterance_no+1]\n",
    "    else:\n",
    "        return 0\n",
    "    \n",
    "def max_q_value(utterance):\n",
    "    max=0\n",
    "    for column in Q_table:\n",
    "        if Q_table.at[utterance,column]>max:\n",
    "            max=Q_table.at[utterance,column]\n",
    "    return max\n",
    "max_strat=''\n",
    "def max_strategy(utterance):\n",
    "    value=''\n",
    "    max=0\n",
    "    for column in Q_table:\n",
    "        if Q_table.at[utterance,column]>max:\n",
    "            max=Q_table.at[utterance,column]\n",
    "            value=column\n",
    "    return value\n",
    "\n",
    "def training(indexes):\n",
    "    \n",
    "    for line in user_utterances:\n",
    "        next_utterance=next_state(indexes)\n",
    "        \n",
    "        prev_value=Q_table.at[user_utterances[indexes],strategies_list[indexes]]\n",
    "        if next_utterance is not 0:\n",
    "            Q_table.at[user_utterances[indexes],strategies_list[indexes]]=((temp_dict[user_utterances[indexes]]-1)*Q_table.at[user_utterances[indexes],strategies_list[indexes]]+Q_table.at[user_utterances[indexes],strategies_list[indexes]]+learning_rate*R_table.at[user_utterances[indexes],strategies_list[indexes]] + discount_factor*max_q_value(next_utterance)-Q_table.at[user_utterances[indexes],strategies_list[indexes]])/temp_dict[user_utterances[indexes]] \n",
    "        else:\n",
    "            Q_table.at[user_utterances[indexes],strategies_list[indexes]]=R_table.at[user_utterances[indexes],strategies_list[indexes]]\n",
    "list1=[]\n",
    "user_utterances=[]\n",
    "bot_strategies=[]\n",
    "\n",
    "def question_or_not(words):\n",
    "    if \"?\" not in words:\n",
    "        q=\"q'\"\n",
    "    else:\n",
    "        q=\"q\"\n",
    "    \n",
    "    return q\n",
    "\n",
    "def count_no_words(words):\n",
    "    no=len(words.split())\n",
    "    if no>=10:\n",
    "        \n",
    "        l=\"l\"\n",
    "    else:\n",
    "        \n",
    "        l=\"l'\"\n",
    "    return l\n",
    "\n",
    "def sentiment(words):\n",
    "    blob = TextBlob(words)\n",
    "    senti=blob.sentiment\n",
    "    if senti.polarity<0.2 and senti.polarity>-0.2:\n",
    "        s=\"s'\"\n",
    "    else:\n",
    "        if senti.polarity>0:\n",
    "            s=\"s+\"\n",
    "        else:\n",
    "            s=\"s-\"\n",
    "    return s\n",
    "    \n",
    "\n",
    "\n",
    "def beginning_end(line,list):\n",
    "    length=len(list)\n",
    "    if list.index(line)==0:\n",
    "        \n",
    "        var=\"b\"\n",
    "    else:\n",
    "        \n",
    "        var=\"e\"\n",
    "    return var\n",
    "\n",
    "def rows_utterance(list1):\n",
    "    \n",
    "    for values in list1:\n",
    "        if question_or_not(values)==\"q\":\n",
    "            if sentiment(values)==\"s+\":\n",
    "                if beginning_end(values,list1)==\"b\":\n",
    "                    result=\"qs+b\"\n",
    "                else:\n",
    "                    result=\"qs-e\"\n",
    "            elif sentiment(values)==\"s-\":\n",
    "                if beginning_end(values,list1)==\"b\":\n",
    "                    result=\"qs-b\"\n",
    "                else:\n",
    "                    result=\"qs-e\"\n",
    "            else:\n",
    "                if beginning_end(values,list1)==\"b\":\n",
    "                    result=\"qs'b\"\n",
    "                else:\n",
    "                    result=\"qs'e\"\n",
    "        else:\n",
    "            if count_no_words(values)==\"l\":\n",
    "            \n",
    "                if sentiment(values)==\"s+\":\n",
    "                \n",
    "                    if beginning_end(values,list1)==\"b\":\n",
    "                        result=\"q'ls+b\"\n",
    "                    else:\n",
    "                        result=\"q'ls+e\"\n",
    "                elif sentiment(values)==\"s-\":\n",
    "                    if beginning_end(values,list1)==\"b\":\n",
    "                        result=\"q'ls-b\"\n",
    "                    else:\n",
    "                        result=\"q'ls-e\"\n",
    "                else:\n",
    "                    if beginning_end(values,list1)==\"b\":\n",
    "                        result=\"q'ls'b\"\n",
    "                    else:\n",
    "                        result=\"q'ls'e\"\n",
    "            else:\n",
    "                if sentiment(values)==\"s+\":\n",
    "                \n",
    "                    if beginning_end(values,list1)==\"b\":\n",
    "                        result=\"q'l's+b\"\n",
    "                    else:\n",
    "                        result=\"q'l's+e\"\n",
    "                elif sentiment(values)==\"s-\":\n",
    "                    if beginning_end(values,list1)==\"b\":\n",
    "                        result=\"q'l's-b\"\n",
    "                    else:\n",
    "                        result=\"q'l's-e\"\n",
    "                else:\n",
    "                    if beginning_end(values,list1)==\"b\":\n",
    "                        result=\"q'l's'b\"\n",
    "                    else:\n",
    "                        result=\"q'l's'e\"\n",
    "        user_utterances.append(result)\n",
    "    return user_utterances\n",
    "\n",
    "def single_utterance(values):\n",
    "    if question_or_not(values)==\"q\":\n",
    "        if sentiment(values)==\"s+\":\n",
    "            result=\"qs+e\"\n",
    "        elif sentiment(values)==\"s-\":\n",
    "            result=\"qs-e\"\n",
    "        else:\n",
    "            result=\"qs'e\"\n",
    "    else:\n",
    "        if count_no_words(values)==\"l\":\n",
    "            \n",
    "            if sentiment(values)==\"s+\":\n",
    "                \n",
    "                result=\"q'ls+e\"\n",
    "            elif sentiment(values)==\"s-\":\n",
    "                result=\"q'ls-e\"\n",
    "            else:\n",
    "                result=\"q'ls'e\"\n",
    "        else:\n",
    "            if sentiment(values)==\"s+\":\n",
    "                \n",
    "                result=\"q'l's+e\"\n",
    "            elif sentiment(values)==\"s-\":\n",
    "                result=\"q'l's-e\"\n",
    "            else:\n",
    "                result=\"q'l's'e\"\n",
    "        \n",
    "    return result\n",
    "\n",
    "#_______________________________________________________________________________________________________________________________\n",
    "#CODE FOR EXTRACTING VALUES FROM REWARD TABLE AND LEARNING FROM IT\n",
    "\n",
    "user_utterances=[]\n",
    "\n",
    "for filename in os.listdir(abs_data_path):\n",
    "    \n",
    "    file = open(abs_data_path+filename,'r')\n",
    "    \n",
    "    strategies_list=[]\n",
    "    user_lines=[]\n",
    "    user_utterances=[]\n",
    "    for lines in file:\n",
    "        \n",
    "        if not lines.startswith((\"overall\",\"engaing\",\"return\",\"interupt\",\"start\")):\n",
    "            if lines.startswith('Bot'):\n",
    "                strategies_list.insert(len(strategies_list),check_strategies(lines))\n",
    "            else:\n",
    "                user_lines.append(get_utterance(lines))\n",
    "                \n",
    "        else:\n",
    "            strategies_list=strategies_list[1:]\n",
    "            if lines.endswith(','):\n",
    "                populate_evaluation(lines)\n",
    "                user_utterances=rows_utterance(user_lines)\n",
    "                user_utterances=user_utterances[:len(user_utterances)-1]\n",
    "                \n",
    "                temp_dict[user_utterances[0]]=temp_dict[user_utterances[0]]+1\n",
    "                R_table.at[user_utterances[0],strategies_list[0]]=((temp_dict[user_utterances[0]]-1)*R_table.at[user_utterances[0],strategies_list[0]]+0.8*float(evaluation_dict['start'])+0.2*float(evaluation_dict['overall']))/temp_dict[user_utterances[0]]\n",
    "                count=1\n",
    "                for line in user_utterances[1:len(user_utterances)]:\n",
    "                    \n",
    "                    temp_dict[user_utterances[count]]=temp_dict[user_utterances[count]]+1\n",
    "                    R_table.at[user_utterances[count],strategies_list[count]]=((temp_dict[user_utterances[count]]-1)*R_table.at[user_utterances[count],strategies_list[count]]+0.4*float(evaluation_dict['engaing'])+0.4*float(evaluation_dict['interupt'])+0.2*float(evaluation_dict['overall']))/temp_dict[user_utterances[count]]               \n",
    "                    count=count+1\n",
    "            else:\n",
    "                pass\n",
    "        \n",
    "    file.close()\n",
    "#_______________________________________________________________________________________________________________________________\n",
    "\n",
    "#for training from the model\n",
    "for filename in os.listdir(abs_data_path):\n",
    "    \n",
    "    file = open(abs_data_path+filename,'r')\n",
    "    strategies_list=[]\n",
    "    user_lines=[]\n",
    "    user_utterances=[]\n",
    "    for lines in file:\n",
    "        \n",
    "        if not lines.startswith((\"overall\",\"engaing\",\"return\",\"interupt\",\"start\")):\n",
    "            if lines.startswith('Bot'):\n",
    "                strategies_list.insert(len(strategies_list),check_strategies(lines))\n",
    "            else:\n",
    "                user_lines.append(get_utterance(lines))\n",
    "    user_utterances=rows_utterance(user_lines)\n",
    "    strategies_list=strategies_list[1:]\n",
    "    \n",
    "    user_utterances=user_utterances[:len(user_utterances)-1]\n",
    "    \n",
    "    for values in user_utterances:\n",
    "        indexes=user_utterances.index(values)\n",
    "        training(indexes)\n",
    "        \n",
    "#_______________________________________________________________________________________________________________________________\n",
    "#for recording new conversations \n",
    "N_TURN = random.choice(range(5,11)) \n",
    "DATA_PATH = abs_data_path+'/new_convo/'\n",
    "strats = pickle.load(open(abs_strategies_path, \"rb\"))\n",
    "strategies_list=[]\n",
    "utterances=[]\n",
    "\n",
    "def main():\n",
    "    record_convo()\n",
    "    while input(\"Do you want to record another conversation? (Y/N)\").lower() == \"y\":\n",
    "        record_convo()\n",
    "        \n",
    "\n",
    "def record_convo():\n",
    "    # load strategies from pickle\n",
    "    \n",
    "    list1=[]\n",
    "    try:\n",
    "        # create file\n",
    "        name = input(\"Before we start: what's your name?\\n\")\n",
    "        date = str(datetime.datetime.today().date())\n",
    "        convo_num = get_convo_num(name)\n",
    "        filename = abs_data_path + \"{0}_{1}_{2}.txt\".format(name.lower(), date, convo_num) \n",
    "        print(filename)\n",
    "        f = open(filename, \"w\")\n",
    "        \n",
    "        # record converstaion\n",
    "        print(\"----------------------------Here it gones--------------------------------\")\n",
    "        #strategy=Q_table.iloc[0].argmax()\n",
    "        strategy = max_beginning()\n",
    "        utterance = random.choice(strats[strategy])\n",
    "        log(\"Bot\", utterance, strategy, f)\n",
    "        strats[strategy].remove(utterance)\n",
    "        strategies_list.append(strategy)\n",
    "        for turn in range(N_TURN-1):\n",
    "            answer=str(input(name + \": \"))\n",
    "            log(name, answer, \"None\",f)\n",
    "            \n",
    "            utter=single_utterance(answer)\n",
    "            strategy=''\n",
    "            #for exploration\n",
    "            if random.random()<epsilon:\n",
    "                strategy = random.choice(list(strats.keys()))\n",
    "            else:\n",
    "                strategy=max_strategy(utter)\n",
    "            utterances.append(utter)\n",
    "            strategies_list.append(strategy)\n",
    "            utterance=random.choice(list(strats[strategy]))\n",
    "            log(\"Bot\", utterance, strategy, f)\n",
    "            strats[strategy].remove(utterance)\n",
    "            \n",
    "            utter=''\n",
    "        answer=str(input(name + \": \"))\n",
    "        log(name, answer, \"None\",f) \n",
    "        #print(strategies_list)\n",
    "        #print(utterances)\n",
    "        for values in utterances:\n",
    "            indexes=utterances.index(values)\n",
    "            training(indexes)\n",
    "        print(\"-----------------------------The end-------------------------------------\")\n",
    "\n",
    "        # evaluation and wrap up\n",
    "        scores = get_evaluation()\n",
    "        for score_type, score_value in scores.items():\n",
    "            f.write(\"{0}={1},\".format(score_type, score_value))\n",
    "        f.close()\n",
    "        print(\"Done!\")\n",
    "    \n",
    "    except:\n",
    "        os.remove(filename)\n",
    "        raise    \n",
    "\n",
    "\n",
    "def get_evaluation():\n",
    "    scores = {}\n",
    "    questions = {\"overall\": \"How is the conveseration overall? (0 is aweful, 5 is amazing)\\n\",\n",
    "                 \"start\": \"How is the begining of the converstaion? (0: weird and out of context, 5: natural)\\n\",\n",
    "                 \"interupt\": \"How is the continuity of the conversation? (0: you get interupted too much, 5: very fluid and coherent)\\n\",\n",
    "                 \"engaing\": \"How engaging or interesting is the conversation? (0 is boring, 5 is very interesting)\\n\",\n",
    "                 \"return\": \"Would you like to talk to this bot again? (0 is not at all, 5 is definitely)\\n\"}\n",
    "    for score_type in questions.keys():\n",
    "        score = input(questions[score_type])\n",
    "        while len(score) > 1 or ord(score) < ord(\"0\") or ord(score) > ord(\"5\"):\n",
    "            score = input(\"Type a integer between [0,5], otherwise I will keep asking\")\n",
    "        scores[score_type] = score\n",
    "    return scores\n",
    "\n",
    "\n",
    "def get_convo_num(name):\n",
    "    try:\n",
    "        files = glob.glob(DATA_PATH + name.lower() + \"_*\")\n",
    "        existing_nums = list(map(int, [i[:-4].split(\"_\")[-1] for i in files]))\n",
    "        return max(existing_nums) + 1\n",
    "    except ValueError:\n",
    "        return 1\n",
    "    \n",
    "\n",
    "def log(role, utter, strategy, file):\n",
    "    if role == \"Bot\":\n",
    "        print(\"Bot says: \", utter)\n",
    "    time = datetime.datetime.now().strftime(\"%H:%M:%S\")\n",
    "    file.write(\"{0}_{1}_{2}:{3}\\n\".format(role, strategy, time, utter))\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    \n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "R_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Q_table"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}