diff --git a/.gitignore b/.gitignore index 1df737eb..8c0cf3aa 100644 --- a/.gitignore +++ b/.gitignore @@ -6,10 +6,10 @@ graph.html ### Rasa evaluation ### .ipynb_checkpoints/ -modules/notebooks/intents/confmat.png -modules/notebooks/intents/hist.png modules/notebooks/intents/models/ -modules/notebooks/intents/errors.json +modules/notebooks/intents/results/ +modules/notebooks/stories/models/ +modules/notebooks/stories/results/ bot/.ipython/ bot/.keras/ bot/.local/ @@ -18,6 +18,9 @@ bot/results/ ### database ### db/ +### Rabbitmq ### +data/ + ### PyCharm ### .idea/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e318965..fb856805 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,13 +56,14 @@ build requirements: - docker:dind script: - docker login -u $DOCKERHUB_USER -p $DOCKERHUB_PASSWORD - - cd ./docker/bot - - ./build-base.sh publish + - cd ./docker + - docker build . -f docker/requirements.Dockerfile -t lappis/botrequirements:boilerplate + - docker push lappis/botrequirements:boilerplate only: refs: - master changes: - - ./docker/bot/requirements.txt + - ./requirements.txt environment: homolog diff --git a/Makefile b/Makefile index 4713c3d8..839e4c8e 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,8 @@ current_dir := $(shell pwd) +clean: + sudo docker-compose down + ############################## BOILERPLATE ############################## first-run: make build @@ -40,14 +43,14 @@ run-analytics: sensible-browser --no-sandbox http://localhost:5601 run-shell: - docker-compose run --service-ports bot make shell + docker-compose run --rm --service-ports bot make shell run-webchat: docker-compose run -d --rm --service-ports bot-webchat sensible-browser --no-sandbox modules/webchat/index.html run-telegram: - docker-compose run -d --rm --service-ports bot make telegram + docker-compose run -d --rm --service-ports bot_telegram make telegram run-notebooks: docker-compose up -d notebooks diff --git a/README.md b/README.md index 31e8df23..dbb1c53a 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,15 @@ Após realizar o [tutorial](/docs/setup_telegram.md) de exportação de todas va **Antes de seguir adiante. Importante:** As variáveis de ambiente são necessárias para o correto funcionamento do bot, por isso não esqueça de exportá-las. +Edite o arquivo **credentials.yml** e descomente as linhas referentes ao telegram: + +```sh +telegram: + access_token: ${TELEGRAM_TOKEN} + verify: ${TELEGRAM_BOT_USERNAME} + webhook_url: ${TELEGRAM_WEBHOOK} +``` + Se ainda não tiver treinado seu bot execute antes: ```sh @@ -87,7 +96,7 @@ Para a visualização dos dados da interação entre o usuário e o chatbot nós * Para uma **configuração rápida** execute o seguinte comando: ```sh -sudo make buil-analytics +sudo make build-analytics ``` O comando acima só precisa ser executado apenas 1 vez e já vai deixar toda a infra de `analytics` pronta para o uso. diff --git a/bot/Makefile b/bot/Makefile index 33aaa637..8b03bc1f 100644 --- a/bot/Makefile +++ b/bot/Makefile @@ -1,7 +1,19 @@ +clean: + rm -rf graph.html results/ models/* + ############################## BOT ############################## +# NLU +train-nlu: + rasa train nlu -vv + +test-nlu: + rasa test nlu -vv + +# CORE train: rasa train -vv --out models/ +# MENSAGEIROS shell: rasa shell -m models/ -vv --endpoints endpoints.yml --port 5004 --cors "*" @@ -15,6 +27,6 @@ webchat: run-api: rasa run -m models/ -vv --endpoints endpoints.yml --enable-api -############################## ACTIONS ############################## +# ACTIONS run-actions: rasa run actions --actions actions -vv diff --git a/docker-compose.yml b/docker-compose.yml index d92951f2..f79e532a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,20 +37,6 @@ services: - ./bot/actions:/bot/actions command: sh -c "make run-actions" - # ============================ Telegram Bot ================================= - # Specific Rasa bot integrated with Telegram. - bot-telegram: - build: - context: . - dockerfile: ./docker/bot.Dockerfile - env_file: - - env/bot-telegram.env - ports: - - 5005:5005 - depends_on: - - actions - command: sh -c "make telegram" - # ============================ WebChat Bot ================================= # Specific Rasa bot integrated with WebChat. bot-webchat: @@ -133,7 +119,7 @@ services: - actions volumes: - ./bot:/bot - command: sh -c "make run-telegram" + command: sh -c "make telegram" # =============================== Notebooks ================================= # Rasa lab to enhance hyperparameters. @@ -145,6 +131,7 @@ services: - env/notebooks.env volumes: - ./modules/notebooks:/work/notebooks + - ./bot/:/bot/ ports: - 8888:8888 diff --git a/docker/actions.Dockerfile b/docker/actions.Dockerfile index 40b61866..9cf75871 100644 --- a/docker/actions.Dockerfile +++ b/docker/actions.Dockerfile @@ -3,7 +3,7 @@ FROM botrequirements COPY ./bot/actions/actions.py /bot/actions/actions.py COPY ./bot/Makefile /bot/Makefile -WORKDIR bot/ +WORKDIR /bot EXPOSE 5055 HEALTHCHECK --interval=300s --timeout=60s --retries=5 \ diff --git a/docker/build-base.sh b/docker/build-base.sh deleted file mode 100755 index 93f39215..00000000 --- a/docker/build-base.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - - -set -o errexit -set -o pipefail -set -o nounset - -base_image=lappis/botrequirements:boilerplate - -# Build base image -docker build . -f docker/requirements.Dockerfile -t $base_image - -# Check if user wants to publish -if [ $# -eq 1 ]; then - if [ "$1" = "publish" ]; then - echo; echo; echo "PUBLISHING IMAGES" - docker push $base_image; - fi -else - echo "Execute 'sh build.sh publish' to publish images" -fi diff --git a/docker/coach.Dockerfile b/docker/coach.Dockerfile index 5d1ec182..625e1f89 100644 --- a/docker/coach.Dockerfile +++ b/docker/coach.Dockerfile @@ -1,4 +1,4 @@ -FROM botrequirements as coach +FROM botrequirements WORKDIR /bot diff --git a/docker/notebooks.Dockerfile b/docker/notebooks.Dockerfile index 9a1f2c4d..386e4ee3 100644 --- a/docker/notebooks.Dockerfile +++ b/docker/notebooks.Dockerfile @@ -1,4 +1,4 @@ -FROM lappis/botrequirements:boilerplate +FROM botrequirements RUN apt-get update && apt-get install -y graphviz libgraphviz-dev pkg-config @@ -6,6 +6,8 @@ RUN apt-get update && apt-get install -y graphviz libgraphviz-dev pkg-config # acording to each OS. because of it it's not added to bot.requirements RUN pip install jupyter pygraphviz==1.5 +COPY ./bot /bot + WORKDIR /work/ CMD jupyter-notebook --allow-root --NotebookApp.token='' --ip=0.0.0.0 --NotebookApp.password='' diff --git a/docker/requirements.Dockerfile b/docker/requirements.Dockerfile index 4291b436..f2dc756d 100644 --- a/docker/requirements.Dockerfile +++ b/docker/requirements.Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6-slim as botrequirements +FROM python:3.6-slim RUN apt update && apt install -y gcc make diff --git a/env/notebooks.env b/env/notebooks.env index 61af9af3..891f698e 100644 --- a/env/notebooks.env +++ b/env/notebooks.env @@ -1,8 +1,5 @@ -COACH_DIR_PATH=../../coach/ -COACH_DOMAIN_PATH=../../coach/domain.yml -COACH_NLU_CONFIG_PATH=../../coach/nlu_config.yml -COACH_STORIES_PATH=../../coach/data/stories/ -COACH_INTENTS_PATH=../../coach/data/intents/ -COACH_MODELS_PATH=/models/ -COACH_MODELS_NLU_PATH=/models/nlu/current/ -COACH_MODELS_DIALOGUE_PATH=/models/dialogue \ No newline at end of file +DIR_PATH=/bot/ +DOMAIN_PATH=/bot/domain.yml +CONFIG_PATH=/bot/config.yml +DATA_PATH=/bot/data/ +MODELS_PATH=/bot/models/ diff --git a/modules/notebooks/intents/intents-analysis.ipynb b/modules/notebooks/intents/intents-analysis.ipynb index ed1a637c..8b2396e6 100644 --- a/modules/notebooks/intents/intents-analysis.ipynb +++ b/modules/notebooks/intents/intents-analysis.ipynb @@ -30,7 +30,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "rasa: 1.2.2\n" + "Rasa: 1.4.1\n" ] } ], @@ -38,7 +38,7 @@ "from IPython.display import IFrame\n", "\n", "import rasa\n", - "print(\"rasa: {}\".format(rasa.__version__))" + "print(\"Rasa: {}\".format(rasa.__version__))" ] }, { @@ -66,50 +66,72 @@ "name": "stdout", "output_type": "stream", "text": [ - "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n", - "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n", - "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n", - "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n", - "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n", - "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n", "\u001b[94mTraining NLU model...\u001b[0m\n", - "2019-09-27 11:47:56 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.training_data.training_data\u001b[0m - Training data stats: \n", - "\t- intent examples: 589 (37 distinct intents)\n", - "\t- Found intents: 'religiao', 'risada', 'onde_voce_mora', 'historia', 'botao', 'hobby', 'filhos', 'bff', 'action_test', 'comida', 'despedir', 'relationship', 'afirmacao_botao', 'time', 'de_onde_voce_eh', 'elogios', 'signo', 'tudo_bem', 'cumprimentar', 'piada', 'o_que_sei_falar', 'cor', 'negar', 'star_wars', 'me', 'playlist', 'genero', 'negar_despedir', 'triste', 'linguagens', 'como_estou', 'diga_mais', 'license', 'out_of_scope', 'negacao_botao', 'esporte', 'filme'\n", + "2019-10-23 01:04:34 \u001b[1;30mINFO \u001b[0m \u001b[34mabsl\u001b[0m - Entry Point [tensor2tensor.envs.tic_tac_toe_env:TicTacToeEnv] registered with id [T2TEnv-TicTacToeEnv-v0]\n", + "2019-10-23 01:04:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 588 (37 distinct intents)\n", + "\t- Found intents: 'license', 'cumprimentar', 'time', 'historia', 'tudo_bem', 'negar', 'afirmacao_botao', 'linguagens', 'despedir', 'relationship', 'filme', 'genero', 'triste', 'negacao_botao', 'elogios', 'bff', 'filhos', 'onde_voce_mora', 'religiao', 'comida', 'diga_mais', 'cor', 'signo', 'piada', 'botao', 'action_test', 'o_que_sei_falar', 'playlist', 'hobby', 'out_of_scope', 'me', 'star_wars', 'esporte', 'de_onde_voce_eh', 'risada', 'negar_despedir', 'como_estou'\n", + "\t- Number of response examples: 0 (0 distinct response)\n", "\t- entity examples: 173 (22 distinct entities)\n", - "\t- found entities: 'how', 'religiao', 'where', 'historia', 'hobby', 'filhos', 'bff', 'comida', 'live', 'relationship', 'signo', 'piada', 'cor', 'me', 'playlist', 'genero', 'triste', 'linguagens', 'license', 'starwars', 'esporte', 'filme'\n", + "\t- found entities: 'starwars', 'license', 'historia', 'where', 'linguagens', 'filme', 'relationship', 'genero', 'triste', 'bff', 'filhos', 'religiao', 'comida', 'cor', 'signo', 'piada', 'how', 'live', 'playlist', 'hobby', 'me', 'esporte'\n", "\n", - "/usr/local/lib/python3.6/site-packages/rasa/nlu/training_data/training_data.py:199: UserWarning: Intent 'afirmacao_botao' has only 1 training examples! Minimum is 2, training may fail.\n", + "/usr/local/lib/python3.6/site-packages/rasa/nlu/training_data/training_data.py:336: UserWarning: Intent 'afirmacao_botao' has only 1 training examples! Minimum is 2, training may fail.\n", " intent, count, self.MIN_EXAMPLES_PER_INTENT\n", - "/usr/local/lib/python3.6/site-packages/rasa/nlu/training_data/training_data.py:199: UserWarning: Intent 'negacao_botao' has only 1 training examples! Minimum is 2, training may fail.\n", + "/usr/local/lib/python3.6/site-packages/rasa/nlu/training_data/training_data.py:336: UserWarning: Intent 'negacao_botao' has only 1 training examples! Minimum is 2, training may fail.\n", " intent, count, self.MIN_EXAMPLES_PER_INTENT\n", - "2019-09-27 11:47:56 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Starting to train component WhitespaceTokenizer\n", - "2019-09-27 11:47:56 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Finished training component.\n", - "2019-09-27 11:47:56 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Starting to train component CRFEntityExtractor\n", - "2019-09-27 11:47:57 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Finished training component.\n", - "2019-09-27 11:47:57 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Starting to train component EntitySynonymMapper\n", - "2019-09-27 11:47:57 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Finished training component.\n", - "2019-09-27 11:47:57 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Starting to train component CountVectorsFeaturizer\n", - "2019-09-27 11:47:57 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Finished training component.\n", - "2019-09-27 11:47:57 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Starting to train component EmbeddingIntentClassifier\n", - "2019-09-27 11:47:58 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.classifiers.embedding_intent_classifier\u001b[0m - Accuracy is updated every 10 epochs\n", - "Epochs: 100%|██████████| 300/300 [00:17<00:00, 17.04it/s, loss=0.067, acc=1.000]\n", - "2019-09-27 11:48:16 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.classifiers.embedding_intent_classifier\u001b[0m - Finished training embedding classifier, loss=0.067, train accuracy=1.000\n", - "2019-09-27 11:48:16 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Finished training component.\n", - "2019-09-27 11:48:16 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Successfully saved model into '/tmp/tmp5g8k70_s/nlu'\n", + "2019-10-23 01:04:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Starting to train component WhitespaceTokenizer\n", + "2019-10-23 01:04:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Finished training component.\n", + "2019-10-23 01:04:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Starting to train component CRFEntityExtractor\n", + "2019-10-23 01:04:35 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Finished training component.\n", + "2019-10-23 01:04:35 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Starting to train component EntitySynonymMapper\n", + "2019-10-23 01:04:35 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Finished training component.\n", + "2019-10-23 01:04:35 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Starting to train component CountVectorsFeaturizer\n", + "2019-10-23 01:04:35 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Finished training component.\n", + "2019-10-23 01:04:35 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Starting to train component EmbeddingIntentClassifier\n", + "Epochs: 100%|██████████| 300/300 [00:15<00:00, 19.24it/s, loss=1.005, acc=1.000]\n", + "2019-10-23 01:04:53 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.utils.train_utils\u001b[0m - Finished training embedding policy, train loss=1.005, train accuracy=1.000\n", + "2019-10-23 01:04:53 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Finished training component.\n", + "2019-10-23 01:04:54 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.model\u001b[0m - Successfully saved model into '/tmp/tmpj0lhn0y2/nlu'\n", "\u001b[94mNLU model training completed.\u001b[0m\n", - "\u001b[92mYour Rasa model is trained and saved at '/work/notebooks/intents/models/nlu-20190927-114816.tar.gz'.\u001b[0m\n" + "\u001b[92mYour Rasa model is trained and saved at '/work/notebooks/intents/models/nlu-20191023-010454.tar.gz'.\u001b[0m\n" ] } ], "source": [ - "!rasa train nlu --config ../../coach/nlu_config.yml --nlu ../../coach/data/intents/" + "!rasa train nlu --config $CONFIG_PATH --nlu $DATA_PATH" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "O comando acima gerou um modelo apenas de **NLU**, este modelo pode ser achado na seguinte pasta:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`modules/notebooks/intents/models/`" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "nlu-20191021-134116.tar.gz nlu-20191023-000948.tar.gz\r\n", + "nlu-20191021-134626.tar.gz nlu-20191023-002148.tar.gz\r\n", + "nlu-20191023-000924.tar.gz nlu-20191023-010454.tar.gz\r\n" + ] + } + ], + "source": [ + "!ls models/" ] }, { @@ -125,7 +147,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "* O comando `!python -m rasa_nlu.test` é a base para a avaliação do chatbot.\n", + "* O comando `rasa test nlu` é a base para a avaliação do chatbot.\n", "\n", "* Na célula abaixo a flag `-h` foi utilizada para mostrar as funções e a forma de uso de cada uma delas, mude seus valores e flags para ter as informações desejadas na sua análise.\n", "\n", @@ -134,7 +156,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": { "scrolled": true }, @@ -143,9 +165,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "usage: rasa test nlu [-h] [-v] [-vv] [--quiet] [-m MODEL] [-u NLU]\r\n", - " [--report [REPORT]] [--successes [SUCCESSES]]\r\n", - " [--errors ERRORS] [--histogram HISTOGRAM]\r\n", + "usage: rasa test nlu [-h] [-v] [-vv] [--quiet] [-m MODEL] [-u NLU] [--out OUT]\r\n", + " [--successes] [--no-errors] [--histogram HISTOGRAM]\r\n", " [--confmat CONFMAT] [-c CONFIG [CONFIG ...]]\r\n", " [--cross-validation] [-f FOLDS] [-r RUNS]\r\n", " [-p PERCENTAGES [PERCENTAGES ...]]\r\n", @@ -158,13 +179,12 @@ " directory. (default: models)\r\n", " -u NLU, --nlu NLU File or folder containing your NLU data. (default:\r\n", " data)\r\n", - " --report [REPORT] Output path to save the intent/entity metrics report.\r\n", - " (default: None)\r\n", - " --successes [SUCCESSES]\r\n", - " Output path to save successful predictions. (default:\r\n", - " None)\r\n", - " --errors ERRORS Output path to save model errors. (default:\r\n", - " errors.json)\r\n", + " --out OUT Output path for any files created during the\r\n", + " evaluation. (default: results)\r\n", + " --successes If set successful predictions (intent and entities)\r\n", + " will be written to a file. (default: False)\r\n", + " --no-errors If set incorrect predictions (intent and entities)\r\n", + " will NOT be written to a file. (default: False)\r\n", " --histogram HISTOGRAM\r\n", " Output path for the confidence histogram. (default:\r\n", " hist.png)\r\n", @@ -218,29 +238,32 @@ "* O comando abaixo gera informações relevates para a validação das `intents` são elas:\n", " * Matriz de confusão\n", " * Histograma\n", - " * Erros de intents" + " * Erros de intents\n", + "\n", + "Essas informações e os demais arquivos podem ser encontrados na pasta `results`." ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "confmat.png errors.json hist.png intents-analysis.ipynb models\r\n" + "CRFEntityExtractor_errors.json\tconfmat.png intent_errors.json\r\n", + "CRFEntityExtractor_report.json\thist.png intent_report.json\r\n" ] } ], "source": [ - "!ls" + "!ls results" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": { "scrolled": true }, @@ -249,79 +272,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "rm: cannot remove 'erros.json': No such file or directory\n", - "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n", - "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n", - "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n", - "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n", - "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n", - "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", - " np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n", - "2019-09-27 11:48:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Running model for predictions:\n", - "100%|████████████████████████████████████████| 589/589 [00:00<00:00, 939.27it/s]\n", - "2019-09-27 11:48:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Intent evaluation results:\n", - "2019-09-27 11:48:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Intent Evaluation: Only considering those 589 examples that have a defined intent out of 589 examples\n", - "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n", - " 'recall', 'true', average, warn_for)\n", - "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n", - " 'recall', 'true', average, warn_for)\n", - "2019-09-27 11:48:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - F1-Score: 0.9991303987742763\n", - "2019-09-27 11:48:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Precision: 1.0\n", - "2019-09-27 11:48:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Accuracy: 0.99830220713073\n", - "2019-09-27 11:48:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Classification report: \n", - " precision recall f1-score support\n", - "\n", - " 0.00 0.00 0.00 0\n", - " action_test 1.00 1.00 1.00 6\n", - "afirmacao_botao 1.00 1.00 1.00 1\n", - " bff 1.00 1.00 1.00 15\n", - " botao 1.00 1.00 1.00 3\n", - " comida 1.00 1.00 1.00 16\n", - " como_estou 1.00 1.00 1.00 15\n", - " cor 1.00 1.00 1.00 13\n", - " cumprimentar 1.00 1.00 1.00 18\n", - "de_onde_voce_eh 1.00 1.00 1.00 14\n", - " despedir 1.00 1.00 1.00 20\n", - " diga_mais 1.00 1.00 1.00 9\n", - " elogios 1.00 1.00 1.00 27\n", - " esporte 1.00 1.00 1.00 20\n", - " filhos 1.00 1.00 1.00 15\n", - " filme 1.00 1.00 1.00 13\n", - " genero 1.00 1.00 1.00 23\n", - " historia 1.00 1.00 1.00 15\n", - " hobby 1.00 1.00 1.00 19\n", - " license 1.00 1.00 1.00 15\n", - " linguagens 1.00 0.95 0.98 21\n", - " me 1.00 1.00 1.00 19\n", - " negacao_botao 1.00 1.00 1.00 1\n", - " negar 1.00 1.00 1.00 11\n", - " negar_despedir 1.00 1.00 1.00 31\n", - "o_que_sei_falar 1.00 1.00 1.00 17\n", - " onde_voce_mora 1.00 1.00 1.00 14\n", - " out_of_scope 1.00 1.00 1.00 11\n", - " piada 1.00 1.00 1.00 12\n", - " playlist 1.00 1.00 1.00 18\n", - " relationship 1.00 1.00 1.00 28\n", - " religiao 1.00 1.00 1.00 24\n", - " risada 1.00 1.00 1.00 19\n", - " signo 1.00 1.00 1.00 9\n", - " star_wars 1.00 1.00 1.00 22\n", - " time 1.00 1.00 1.00 14\n", - " triste 1.00 1.00 1.00 18\n", - " tudo_bem 1.00 1.00 1.00 23\n", - "\n", - " micro avg 1.00 1.00 1.00 589\n", - " macro avg 0.97 0.97 0.97 589\n", - " weighted avg 1.00 1.00 1.00 589\n", - "\n", - "2019-09-27 11:48:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Model prediction errors saved to errors.json.\n", - "2019-09-27 11:48:22 \u001b[1;30mINFO \u001b[0m \u001b[34mmatplotlib.font_manager\u001b[0m - generated new fontManager\n", - "2019-09-27 11:48:23 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Confusion matrix, without normalization: \n", + "2019-10-23 01:05:05 \u001b[1;30mINFO \u001b[0m \u001b[34mabsl\u001b[0m - Entry Point [tensor2tensor.envs.tic_tac_toe_env:TicTacToeEnv] registered with id [T2TEnv-TicTacToeEnv-v0]\n", + "2019-10-23 01:05:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Running model for predictions:\n", + "100%|████████████████████████████████████████| 588/588 [00:00<00:00, 730.43it/s]\n", + "2019-10-23 01:05:07 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Intent evaluation results:\n", + "2019-10-23 01:05:07 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Intent Evaluation: Only considering those 588 examples that have a defined intent out of 588 examples\n", + "2019-10-23 01:05:07 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Classification report saved to results/intent_report.json.\n", + "2019-10-23 01:05:07 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Incorrect intent predictions saved to results/intent_errors.json.\n", + "2019-10-23 01:05:07 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Confusion matrix, without normalization: \n", "[[ 0 0 0 ... 0 0 0]\n", " [ 0 6 0 ... 0 0 0]\n", " [ 0 0 1 ... 0 0 0]\n", @@ -329,51 +287,16 @@ " [ 0 0 0 ... 14 0 0]\n", " [ 0 0 0 ... 0 18 0]\n", " [ 0 0 0 ... 0 0 23]]\n", - "Figure(2000x2000)\n", - "Figure(1000x1000)\n", - "2019-09-27 11:48:27 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Entity evaluation results:\n", - "2019-09-27 11:48:27 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Evaluation for entity extractor: CRFEntityExtractor \n", - "2019-09-27 11:48:27 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - F1-Score: 0.9838414578944666\n", - "2019-09-27 11:48:27 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Precision: 0.9856499702361066\n", - "2019-09-27 11:48:27 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Accuracy: 0.9850299401197605\n", - "2019-09-27 11:48:27 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Classification report: \n", - " precision recall f1-score support\n", - "\n", - " bff 1.00 1.00 1.00 17\n", - " comida 1.00 1.00 1.00 17\n", - " cor 1.00 1.00 1.00 13\n", - " esporte 1.00 1.00 1.00 6\n", - " filhos 1.00 1.00 1.00 11\n", - " filme 0.80 1.00 0.89 4\n", - " genero 1.00 1.00 1.00 16\n", - " historia 1.00 0.83 0.91 6\n", - " hobby 1.00 0.45 0.62 11\n", - " how 1.00 0.88 0.93 24\n", - " license 1.00 0.86 0.92 14\n", - " linguagens 1.00 1.00 1.00 14\n", - " live 1.00 1.00 1.00 13\n", - " me 1.00 1.00 1.00 6\n", - " no_entity 0.99 1.00 0.99 1433\n", - " piada 0.78 1.00 0.88 7\n", - " playlist 0.88 1.00 0.93 7\n", - "relationship 1.00 0.89 0.94 9\n", - " religiao 0.94 1.00 0.97 16\n", - " signo 1.00 1.00 1.00 3\n", - " starwars 1.00 0.69 0.82 13\n", - " triste 1.00 0.86 0.92 7\n", - " where 1.00 0.33 0.50 3\n", - "\n", - " micro avg 0.99 0.99 0.99 1670\n", - " macro avg 0.97 0.90 0.92 1670\n", - "weighted avg 0.99 0.99 0.98 1670\n", - "\n" + "2019-10-23 01:05:13 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Entity evaluation results:\n", + "2019-10-23 01:05:13 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Evaluation for entity extractor: CRFEntityExtractor \n", + "2019-10-23 01:05:13 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Classification report for 'CRFEntityExtractor' saved to 'results/CRFEntityExtractor_report.json'.\n", + "2019-10-23 01:05:13 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Incorrect entity predictions saved to results/CRFEntityExtractor_errors.json.\n" ] } ], "source": [ - "!rm erros.json\n", - "# !python -m rasa_nlu.test -d $COACH_INTENTS_PATH -m $COACH_MODELS_NLU_PATH --mode evaluation\n", - "!rasa test nlu --nlu ../../coach/data/intents/ --model ./models/" + "!rm -f resutls/*erros.json\n", + "!rasa test nlu --nlu $DATA_PATH --model ./models/" ] }, { @@ -383,14 +306,12 @@ "#### Matriz de Confusão\n", "* A matriz de confusão mostra a correlação entre as intents.\n", "* A diagonal principal tem forte correlação pois mostra a relação de uma intent **com ela mesma**\n", - "* O ideal é que não haja **nenhum valor** diferente de **0 fora da diagonal principal**.\n", - "\n", - "`Execute este notebook no Firefox para visualizar o PDF da matriz de confusão`" + "* O ideal é que não haja **nenhum valor** diferente de **0 fora da diagonal principal**." ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": { "scrolled": false }, @@ -402,23 +323,23 @@ " \n", " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "IFrame(src='./confmat.png', width=900, height=700)" + "IFrame(src='./results/confmat.png', width=900, height=700)" ] }, { @@ -433,7 +354,25 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CRFEntityExtractor_errors.json\tconfmat.png intent_errors.json\r\n", + "CRFEntityExtractor_report.json\thist.png intent_report.json\r\n" + ] + } + ], + "source": [ + "!ls results/" + ] + }, + { + "cell_type": "code", + "execution_count": 9, "metadata": { "scrolled": true }, @@ -456,7 +395,227 @@ } ], "source": [ - "%cat errors.json" + "!cat results/intent_errors.json" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[\r\n", + " {\r\n", + " \"text\": \"qual é o seu deus?\",\r\n", + " \"entities\": [],\r\n", + " \"predicted_entities\": [\r\n", + " {\r\n", + " \"start\": 13,\r\n", + " \"end\": 17,\r\n", + " \"value\": \"deus\",\r\n", + " \"entity\": \"religiao\",\r\n", + " \"confidence\": 0.474498353384622,\r\n", + " \"extractor\": \"CRFEntityExtractor\"\r\n", + " }\r\n", + " ]\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"concelhos do yoda\",\r\n", + " \"entities\": [\r\n", + " {\r\n", + " \"start\": 13,\r\n", + " \"end\": 17,\r\n", + " \"value\": \"yoda\",\r\n", + " \"entity\": \"starwars\"\r\n", + " }\r\n", + " ],\r\n", + " \"predicted_entities\": []\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"guerra nas estrelas\",\r\n", + " \"entities\": [\r\n", + " {\r\n", + " \"start\": 0,\r\n", + " \"end\": 19,\r\n", + " \"value\": \"guerra nas estrelas\",\r\n", + " \"entity\": \"starwars\"\r\n", + " }\r\n", + " ],\r\n", + " \"predicted_entities\": []\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"como você é?\",\r\n", + " \"entities\": [\r\n", + " {\r\n", + " \"start\": 0,\r\n", + " \"end\": 11,\r\n", + " \"value\": \"como você é\",\r\n", + " \"entity\": \"how\"\r\n", + " }\r\n", + " ],\r\n", + " \"predicted_entities\": []\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"me conta uma piada\",\r\n", + " \"entities\": [],\r\n", + " \"predicted_entities\": [\r\n", + " {\r\n", + " \"start\": 13,\r\n", + " \"end\": 18,\r\n", + " \"value\": \"piada\",\r\n", + " \"entity\": \"piada\",\r\n", + " \"confidence\": 0.6223906350689047,\r\n", + " \"extractor\": \"CRFEntityExtractor\"\r\n", + " }\r\n", + " ]\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"me fala uma piada\",\r\n", + " \"entities\": [],\r\n", + " \"predicted_entities\": [\r\n", + " {\r\n", + " \"start\": 12,\r\n", + " \"end\": 17,\r\n", + " \"value\": \"piada\",\r\n", + " \"entity\": \"piada\",\r\n", + " \"confidence\": 0.6187680883976037,\r\n", + " \"extractor\": \"CRFEntityExtractor\"\r\n", + " }\r\n", + " ]\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"voce e um software livre\",\r\n", + " \"entities\": [\r\n", + " {\r\n", + " \"start\": 10,\r\n", + " \"end\": 24,\r\n", + " \"value\": \"software livre\",\r\n", + " \"entity\": \"license\"\r\n", + " }\r\n", + " ],\r\n", + " \"predicted_entities\": []\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"me indica uma playlist\",\r\n", + " \"entities\": [],\r\n", + " \"predicted_entities\": [\r\n", + " {\r\n", + " \"start\": 14,\r\n", + " \"end\": 22,\r\n", + " \"value\": \"playlist\",\r\n", + " \"entity\": \"playlist\",\r\n", + " \"confidence\": 0.8034555766584158,\r\n", + " \"extractor\": \"CRFEntityExtractor\"\r\n", + " }\r\n", + " ]\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"onde você nasceu?\",\r\n", + " \"entities\": [\r\n", + " {\r\n", + " \"start\": 10,\r\n", + " \"end\": 16,\r\n", + " \"value\": \"nasceu\",\r\n", + " \"entity\": \"where\"\r\n", + " }\r\n", + " ],\r\n", + " \"predicted_entities\": []\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"onde voce nasceu?\",\r\n", + " \"entities\": [\r\n", + " {\r\n", + " \"start\": 10,\r\n", + " \"end\": 16,\r\n", + " \"value\": \"nasceu\",\r\n", + " \"entity\": \"where\"\r\n", + " }\r\n", + " ],\r\n", + " \"predicted_entities\": []\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"Você tem namorado?\",\r\n", + " \"entities\": [\r\n", + " {\r\n", + " \"start\": 9,\r\n", + " \"end\": 17,\r\n", + " \"value\": \"namorado\",\r\n", + " \"entity\": \"relationship\"\r\n", + " }\r\n", + " ],\r\n", + " \"predicted_entities\": []\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"melhor filme\",\r\n", + " \"entities\": [],\r\n", + " \"predicted_entities\": [\r\n", + " {\r\n", + " \"start\": 7,\r\n", + " \"end\": 12,\r\n", + " \"value\": \"filme\",\r\n", + " \"entity\": \"filme\",\r\n", + " \"confidence\": 0.5262284272303718,\r\n", + " \"extractor\": \"CRFEntityExtractor\"\r\n", + " }\r\n", + " ]\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"Estou muito triste\",\r\n", + " \"entities\": [\r\n", + " {\r\n", + " \"start\": 12,\r\n", + " \"end\": 18,\r\n", + " \"value\": \"triste\",\r\n", + " \"entity\": \"triste\"\r\n", + " }\r\n", + " ],\r\n", + " \"predicted_entities\": []\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"O que você gosta de fazer?\",\r\n", + " \"entities\": [\r\n", + " {\r\n", + " \"start\": 11,\r\n", + " \"end\": 25,\r\n", + " \"value\": \"gosta de fazer\",\r\n", + " \"entity\": \"hobby\"\r\n", + " }\r\n", + " ],\r\n", + " \"predicted_entities\": []\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"Me fala o que você gosta de fazer?\",\r\n", + " \"entities\": [\r\n", + " {\r\n", + " \"start\": 19,\r\n", + " \"end\": 33,\r\n", + " \"value\": \"gosta de fazer\",\r\n", + " \"entity\": \"hobby\"\r\n", + " }\r\n", + " ],\r\n", + " \"predicted_entities\": []\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"conta pra mim uma historinha\",\r\n", + " \"entities\": [\r\n", + " {\r\n", + " \"start\": 18,\r\n", + " \"end\": 28,\r\n", + " \"value\": \"historinha\",\r\n", + " \"entity\": \"historia\"\r\n", + " }\r\n", + " ],\r\n", + " \"predicted_entities\": []\r\n", + " }\r\n", + "]" + ] + } + ], + "source": [ + "!cat results/CRFEntityExtractor_errors.json" ] }, { @@ -470,7 +629,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 11, "metadata": { "scrolled": false }, @@ -482,23 +641,23 @@ " \n", " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 8, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "IFrame(src='./hist.png', width=900, height=700)" + "IFrame(src='./results/hist.png', width=900, height=700)" ] }, { diff --git a/modules/notebooks/stories/stories-analysis.ipynb b/modules/notebooks/stories/stories-analysis.ipynb index e806cbd2..fc17054b 100644 --- a/modules/notebooks/stories/stories-analysis.ipynb +++ b/modules/notebooks/stories/stories-analysis.ipynb @@ -28,18 +28,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "Versões utilizadas neste jupyter notebook:\n", - "rasa_nlu: 0.15.0 rasa_core: 0.14.1\n" + "Rasa: 1.4.1\n" ] } ], "source": [ - "import rasa_nlu\n", - "import rasa_core\n", "from IPython.display import IFrame\n", "\n", - "print(\"Versões utilizadas neste jupyter notebook:\")\n", - "print(\"rasa_nlu: {} rasa_core: {}\".format(rasa_nlu.__version__, rasa_core.__version__))\n" + "import rasa\n", + "print(\"Rasa: {}\".format(rasa.__version__))" ] }, { @@ -75,24 +72,36 @@ "name": "stdout", "output_type": "stream", "text": [ - "/usr/local/lib/python3.6/runpy.py:125: RuntimeWarning: 'rasa_core.visualize' found in sys.modules after import of package 'rasa_core', but prior to execution of 'rasa_core.visualize'; this may result in unpredictable behaviour\n", - " warn(RuntimeWarning(msg))\n", - "2019-09-24 18:49:01 \u001b[1;30mINFO \u001b[0m \u001b[34mapscheduler.scheduler\u001b[0m - Scheduler started\n", - "\n", - "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n", - "For more information, please see:\n", - " * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n", - " * https://github.com/tensorflow/addons\n", - "If you depend on functionality not listed there, please file an issue.\n", - "\n", - "2019-09-24 18:49:01 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Starting to visualize stories...\n", - "Processed Story Blocks: 100%|███| 62/62 [00:00<00:00, 4766.25it/s, # trackers=1]\n", - "2019-09-24 18:49:02 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Finished graph creation. Saved into file:///work/notebooks/stories/chat_graph.html\n" + "2019-10-23 01:04:23 \u001b[1;30mINFO \u001b[0m \u001b[34mabsl\u001b[0m - Entry Point [tensor2tensor.envs.tic_tac_toe_env:TicTacToeEnv] registered with id [T2TEnv-TicTacToeEnv-v0]\n", + "2019-10-23 01:04:23 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.core.visualize\u001b[0m - Starting to visualize stories...\n", + "Processed Story Blocks: 100%|███| 64/64 [00:00<00:00, 5932.80it/s, # trackers=1]\n", + "Traceback (most recent call last):\n", + " File \"/usr/local/bin/rasa\", line 8, in \n", + " sys.exit(main())\n", + " File \"/usr/local/lib/python3.6/site-packages/rasa/__main__.py\", line 76, in main\n", + " cmdline_arguments.func(cmdline_arguments)\n", + " File \"/usr/local/lib/python3.6/site-packages/rasa/cli/visualize.py\", line 38, in visualize_stories\n", + " args.config, args.domain, args.stories, args.nlu, args.out, args.max_history\n", + " File \"uvloop/loop.pyx\", line 1417, in uvloop.loop.Loop.run_until_complete\n", + " File \"/usr/local/lib/python3.6/site-packages/rasa/core/visualize.py\", line 53, in visualize\n", + " stories_path, output_path, max_history, nlu_training_data=nlu_data_path\n", + " File \"/usr/local/lib/python3.6/site-packages/rasa/core/agent.py\", line 805, in visualize\n", + " fontsize,\n", + " File \"/usr/local/lib/python3.6/site-packages/rasa/core/training/visualization.py\", line 577, in visualize_stories\n", + " fontsize=fontsize,\n", + " File \"/usr/local/lib/python3.6/site-packages/rasa/core/training/visualization.py\", line 486, in visualize_neighborhood\n", + " _merge_equivalent_nodes(graph, max_history)\n", + " File \"/usr/local/lib/python3.6/site-packages/rasa/core/training/visualization.py\", line 207, in _merge_equivalent_nodes\n", + " graph, i, j, max_history\n", + " File \"/usr/local/lib/python3.6/site-packages/rasa/core/training/visualization.py\", line 147, in _nodes_are_equivalent\n", + " return graph.node[node_a][\"label\"] == graph.node[node_b][\"label\"] and (\n", + "AttributeError: 'MultiDiGraph' object has no attribute 'node'\n" ] } ], "source": [ - "!python -m rasa_core.visualize -d $COACH_DOMAIN_PATH -s $COACH_STORIES_PATH -o chat_graph.html" + "# !python -m rasa_core.visualize -d $COACH_DOMAIN_PATH -s $COACH_STORIES_PATH -o chat_graph.html\n", + "!rasa visualize --domain $DOMAIN_PATH --stories $DATA_PATH --config $CONFIG_PATH --out chat_graph.html" ] }, { @@ -125,7 +134,7 @@ " " ], "text/plain": [ - "" + "" ] }, "execution_count": 3, @@ -164,235 +173,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "make: Entering directory '/work/coach'\n", - "python3 -m rasa_nlu.train -c nlu_config.yml --fixed_model_name current \\\n", - "--data data/intents/ -o /src_models --project nlu --verbose\n", - "/usr/local/lib/python3.6/runpy.py:125: RuntimeWarning: 'rasa_nlu.train' found in sys.modules after import of package 'rasa_nlu', but prior to execution of 'rasa_nlu.train'; this may result in unpredictable behaviour\n", - " warn(RuntimeWarning(msg))\n", - "2019-09-24 18:49:05 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/aleatorio.md is md\n", - "2019-09-24 18:49:05 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", - "\t- intent examples: 411 (24 distinct intents)\n", - "\t- Found intents: 'religiao', 'star_wars', 'piada', 'hobby', 'historia', 'filme', 'playlist', 'de_onde_voce_eh', 'genero', 'esporte', 'risada', 'license', 'time', 'me', 'como_estou', 'cor', 'filhos', 'comida', 'onde_voce_mora', 'linguagens', 'relationship', 'signo', 'bff', 'triste'\n", - "\t- entity examples: 173 (22 distinct entities)\n", - "\t- found entities: 'religiao', 'piada', 'esporte', 'cor', 'linguagens', 'relationship', 'triste', 'how', 'starwars', 'where', 'historia', 'hobby', 'live', 'filme', 'playlist', 'license', 'me', 'filhos', 'comida', 'signo', 'bff', 'genero'\n", - "\n", - "2019-09-24 18:49:05 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/actions.md is md\n", - "2019-09-24 18:49:05 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", - "\t- intent examples: 6 (1 distinct intents)\n", - "\t- Found intents: 'action_test'\n", - "\t- entity examples: 0 (0 distinct entities)\n", - "\t- found entities: \n", - "\n", - "2019-09-24 18:49:05 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/geral.md is md\n", - "2019-09-24 18:49:05 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", - "\t- intent examples: 155 (11 distinct intents)\n", - "\t- Found intents: 'out_of_scope', 'elogios', 'cumprimentar', 'negar_despedir', 'negacao_botao', 'negar', 'botao', 'despedir', 'tudo_bem', 'afirmacao_botao', 'diga_mais'\n", - "\t- entity examples: 0 (0 distinct entities)\n", - "\t- found entities: \n", - "\n", - "2019-09-24 18:49:05 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", - "\t- intent examples: 572 (36 distinct intents)\n", - "\t- Found intents: 'religiao', 'piada', 'negar', 'afirmacao_botao', 'esporte', 'cumprimentar', 'como_estou', 'cor', 'negar_despedir', 'onde_voce_mora', 'botao', 'linguagens', 'despedir', 'relationship', 'triste', 'elogios', 'star_wars', 'hobby', 'historia', 'negacao_botao', 'filme', 'playlist', 'de_onde_voce_eh', 'action_test', 'risada', 'license', 'time', 'out_of_scope', 'me', 'filhos', 'comida', 'diga_mais', 'tudo_bem', 'signo', 'bff', 'genero'\n", - "\t- entity examples: 173 (22 distinct entities)\n", - "\t- found entities: 'religiao', 'piada', 'esporte', 'cor', 'linguagens', 'relationship', 'triste', 'how', 'starwars', 'where', 'historia', 'hobby', 'live', 'filme', 'playlist', 'license', 'me', 'filhos', 'comida', 'signo', 'bff', 'genero'\n", - "\n", - "/usr/local/lib/python3.6/site-packages/rasa_nlu/training_data/training_data.py:176: UserWarning: Intent 'afirmacao_botao' has only 1 training examples! Minimum is 2, training may fail.\n", - " self.MIN_EXAMPLES_PER_INTENT))\n", - "/usr/local/lib/python3.6/site-packages/rasa_nlu/training_data/training_data.py:176: UserWarning: Intent 'negacao_botao' has only 1 training examples! Minimum is 2, training may fail.\n", - " self.MIN_EXAMPLES_PER_INTENT))\n", - "2019-09-24 18:49:05 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component WhitespaceTokenizer\n", - "2019-09-24 18:49:05 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-09-24 18:49:05 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CRFEntityExtractor\n", - "2019-09-24 18:49:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-09-24 18:49:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EntitySynonymMapper\n", - "2019-09-24 18:49:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-09-24 18:49:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CountVectorsFeaturizer\n", - "2019-09-24 18:49:07 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-09-24 18:49:07 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EmbeddingIntentClassifier\n", - "\n", - "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n", - "For more information, please see:\n", - " * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n", - " * https://github.com/tensorflow/addons\n", - "If you depend on functionality not listed there, please file an issue.\n", - "\n", - "2019-09-24 18:49:07 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:285: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Use keras.layers.dense instead.\u001b[0m\n", - "2019-09-24 18:49:07 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Colocations handled automatically by placer.\u001b[0m\n", - "2019-09-24 18:49:07 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:286: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Use keras.layers.dropout instead.\u001b[0m\n", - "2019-09-24 18:49:07 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\u001b[0m\n", - "2019-09-24 18:49:08 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Use tf.cast instead.\u001b[0m\n", - "2019-09-24 18:49:08 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Deprecated in favor of operator or tf.math.divide.\u001b[0m\n", - "2019-09-24 18:49:08.516966: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", - "2019-09-24 18:49:08.520763: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2400000000 Hz\n", - "2019-09-24 18:49:08.521012: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55bda5171710 executing computations on platform Host. Devices:\n", - "2019-09-24 18:49:08.521042: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", - "2019-09-24 18:49:08 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Accuracy is updated every 10 epochs\n", - "Epochs: 100%|██████████| 300/300 [00:19<00:00, 14.64it/s, loss=0.066, acc=1.000]\n", - "2019-09-24 18:49:27 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Finished training embedding classifier, loss=0.066, train accuracy=1.000\n", - "2019-09-24 18:49:27 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-09-24 18:49:27 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Successfully saved model into '/src_models/nlu/current'\n", - "2019-09-24 18:49:27 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Finished training\n", - "python3 train.py\n", - "2019-09-24 18:49:30 \u001b[1;30mINFO \u001b[0m \u001b[34mapscheduler.scheduler\u001b[0m - Scheduler started\n", - "2019-09-24 18:49:30 \u001b[1;30mDEBUG \u001b[0m \u001b[34mapscheduler.scheduler\u001b[0m - Looking for jobs to run\n", - "2019-09-24 18:49:30 \u001b[1;30mDEBUG \u001b[0m \u001b[34mapscheduler.scheduler\u001b[0m - No jobs; waiting until a job is added\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n", - "For more information, please see:\n", - " * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n", - " * https://github.com/tensorflow/addons\n", - "If you depend on functionality not listed there, please file an issue.\n", - "\n", - "2019-09-24 18:49:31 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Number of augmentation rounds is 3\n", - "2019-09-24 18:49:31 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Starting data generation round 0 ... (with 1 trackers)\n", - "Processed Story Blocks: 100%|███| 62/62 [00:00<00:00, 4881.03it/s, # trackers=1]\n", - "2019-09-24 18:49:31 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Finished phase (62 training samples found).\n", - "2019-09-24 18:49:31 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Data generation rounds finished.\n", - "2019-09-24 18:49:31 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Found 0 unused checkpoints\n", - "2019-09-24 18:49:31 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Starting augmentation round 0 ... (with 50 trackers)\n", - "Processed Story Blocks: 100%|███| 62/62 [00:00<00:00, 127.26it/s, # trackers=50]\n", - "2019-09-24 18:49:31 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Finished phase (3135 training samples found).\n", - "2019-09-24 18:49:31 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Starting augmentation round 1 ... (with 50 trackers)\n", - "Processed Story Blocks: 100%|████| 62/62 [00:00<00:00, 80.99it/s, # trackers=50]\n", - "2019-09-24 18:49:32 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Finished phase (6065 training samples found).\n", - "2019-09-24 18:49:32 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Starting augmentation round 2 ... (with 50 trackers)\n", - "Processed Story Blocks: 100%|████| 62/62 [00:00<00:00, 76.93it/s, # trackers=50]\n", - "2019-09-24 18:49:33 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Finished phase (9103 training samples found).\n", - "2019-09-24 18:49:33 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Found 9103 training trackers.\n", - "2019-09-24 18:49:33 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Subsampled to 500 augmented training trackers.\n", - "2019-09-24 18:49:33 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - There are 62 original trackers.\n", - "2019-09-24 18:49:33 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.agent\u001b[0m - Agent trainer got kwargs: {'validation_split': 0.2}\n", - "2019-09-24 18:49:33 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.featurizers\u001b[0m - Creating states and action examples from collected trackers (by FullDialogueTrackerFeaturizer(LabelTokenizerSingleStateFeaturizer))...\n", - "Processed trackers: 100%|██████████████████| 562/562 [00:00<00:00, 19604.12it/s]\n", - "2019-09-24 18:49:33 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.featurizers\u001b[0m - The longest dialogue has 21 actions.\n", - "2019-09-24 18:49:33.785780: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", - "2019-09-24 18:49:33.790445: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2400000000 Hz\n", - "2019-09-24 18:49:33.790744: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x5612c9ca5950 executing computations on platform Host. Devices:\n", - "2019-09-24 18:49:33.790804: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", - "2019-09-24 18:49:33 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Colocations handled automatically by placer.\u001b[0m\n", - "2019-09-24 18:49:33 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/keras/backend.py:4010: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\u001b[0m\n", - "_________________________________________________________________\n", - "Layer (type) Output Shape Param # \n", - "=================================================================\n", - "masking (Masking) (None, None, 107) 0 \n", - "_________________________________________________________________\n", - "lstm (LSTM) (None, None, 32) 17920 \n", - "_________________________________________________________________\n", - "time_distributed (TimeDistri (None, None, 43) 1419 \n", - "_________________________________________________________________\n", - "activation (Activation) (None, None, 43) 0 \n", - "=================================================================\n", - "Total params: 19,339\n", - "Trainable params: 19,339\n", - "Non-trainable params: 0\n", - "_________________________________________________________________\n", - "2019-09-24 18:49:34 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.policies.keras_policy\u001b[0m - None\n", - "2019-09-24 18:49:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_core.policies.keras_policy\u001b[0m - Fitting model with 562 total samples and a validation split of 0.1\n", - "2019-09-24 18:49:34 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.policies.policy\u001b[0m - Parameters ignored by `model.fit(...)`: {}\n", - "2019-09-24 18:49:34 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Use tf.cast instead.\u001b[0m\n", - "2019-09-24 18:49:34 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Deprecated in favor of operator or tf.math.divide.\u001b[0m\n", - "Epoch 1/25\n", - "562/562 [==============================] - 2s 3ms/sample - loss: 3.0348 - acc: 0.4113\n", - "Epoch 2/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 2.3769 - acc: 0.4484\n", - "Epoch 3/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 2.1256 - acc: 0.4487\n", - "Epoch 4/25\n", - "562/562 [==============================] - 1s 1ms/sample - loss: 1.8606 - acc: 0.5183\n", - "Epoch 5/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 1.6184 - acc: 0.6657\n", - "Epoch 6/25\n", - "562/562 [==============================] - 2s 3ms/sample - loss: 1.4131 - acc: 0.7401\n", - "Epoch 7/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 1.2552 - acc: 0.7478\n", - "Epoch 8/25\n", - "562/562 [==============================] - 1s 1ms/sample - loss: 1.1183 - acc: 0.7511\n", - "Epoch 9/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 0.9962 - acc: 0.7647\n", - "Epoch 10/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 0.8734 - acc: 0.7997\n", - "Epoch 11/25\n", - "562/562 [==============================] - 1s 1ms/sample - loss: 0.7620 - acc: 0.8326\n", - "Epoch 12/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 0.6538 - acc: 0.8738\n", - "Epoch 13/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 0.5754 - acc: 0.9034\n", - "Epoch 14/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 0.4900 - acc: 0.9331\n", - "Epoch 15/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 0.4061 - acc: 0.9517\n", - "Epoch 16/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 0.3482 - acc: 0.9577\n", - "Epoch 17/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 0.3006 - acc: 0.9605\n", - "Epoch 18/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 0.2479 - acc: 0.9655\n", - "Epoch 19/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 0.1997 - acc: 0.9733\n", - "Epoch 20/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 0.1754 - acc: 0.9753\n", - "Epoch 21/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 0.1625 - acc: 0.9781\n", - "Epoch 22/25\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "562/562 [==============================] - 1s 2ms/sample - loss: 0.1336 - acc: 0.9807\n", - "Epoch 23/25\n", - "562/562 [==============================] - 1s 2ms/sample - loss: 0.1300 - acc: 0.9801\n", - "Epoch 24/25\n", - "562/562 [==============================] - 1s 1ms/sample - loss: 0.1092 - acc: 0.9831\n", - "Epoch 25/25\n", - "562/562 [==============================] - 1s 1ms/sample - loss: 0.1098 - acc: 0.9822\n", - "2019-09-24 18:50:00 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_core.policies.keras_policy\u001b[0m - Done fitting keras policy model\n", - "2019-09-24 18:50:00 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.featurizers\u001b[0m - Creating states and action examples from collected trackers (by MaxHistoryTrackerFeaturizer(SingleStateFeaturizer))...\n", - "Processed trackers: 100%|██████| 62/62 [00:00<00:00, 3132.23it/s, # actions=125]\n", - "2019-09-24 18:50:00 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.featurizers\u001b[0m - Created 125 action examples.\n", - "Processed actions: 125it [00:00, 10543.75it/s, # examples=125]\n", - "2019-09-24 18:50:00 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.policies.memoization\u001b[0m - Memorized 125 unique examples.\n", - "2019-09-24 18:50:00 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_core.agent\u001b[0m - Persisted model to '/src_models/dialogue'\n", - "make: Leaving directory '/work/coach'\n" + "\u001b[92mNothing changed. You can use the old model stored at '/work/notebooks/stories/models/20191023-005105.tar.gz'.\u001b[0m\r\n" ] } ], "source": [ - "!make train -C $COACH_DIR_PATH" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* `rasa_core.evaluate` é o método utilizado para verificar a qualidade das suas `stories`." + "!rasa train --config $CONFIG_PATH --domain $DOMAIN_PATH --data $DATA_PATH" ] }, { @@ -406,130 +192,139 @@ "name": "stdout", "output_type": "stream", "text": [ - "Calling `rasa_core.evaluate` is deprecated. Please use `rasa_core.test` instead.\n", - "\n", - "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n", - "For more information, please see:\n", - " * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n", - " * https://github.com/tensorflow/addons\n", - "If you depend on functionality not listed there, please file an issue.\n", - "\n", - "2019-09-24 18:50:05.218174: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", - "2019-09-24 18:50:05.222123: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2400000000 Hz\n", - "2019-09-24 18:50:05.222347: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55db1dceb5f0 executing computations on platform Host. Devices:\n", - "2019-09-24 18:50:05.222371: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", - "WARNING:tensorflow:From /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Colocations handled automatically by placer.\n", - "WARNING:tensorflow:From /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Colocations handled automatically by placer.\n", - "WARNING:tensorflow:From /usr/local/lib/python3.6/site-packages/tensorflow/python/keras/backend.py:4010: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n", - "WARNING:tensorflow:From /usr/local/lib/python3.6/site-packages/tensorflow/python/keras/backend.py:4010: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n", - "WARNING:tensorflow:From /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Use tf.cast instead.\n", - "WARNING:tensorflow:From /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Use tf.cast instead.\n", - "WARNING:tensorflow:From /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Deprecated in favor of operator or tf.math.divide.\n", - "WARNING:tensorflow:From /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Deprecated in favor of operator or tf.math.divide.\n", - "Processed Story Blocks: 100%|███| 62/62 [00:00<00:00, 5293.47it/s, # trackers=1]\n", - "INFO:rasa_core.test:Evaluating 62 stories\n", + "2019-10-23 01:04:37 \u001b[1;30mINFO \u001b[0m \u001b[34mabsl\u001b[0m - Entry Point [tensor2tensor.envs.tic_tac_toe_env:TicTacToeEnv] registered with id [T2TEnv-TicTacToeEnv-v0]\n", + "Processed Story Blocks: 100%|███| 64/64 [00:00<00:00, 4843.31it/s, # trackers=1]\n", + "2019-10-23 01:04:37 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.core.test\u001b[0m - Evaluating 64 stories\n", "Progress:\n", - "100%|███████████████████████████████████████████| 62/62 [00:01<00:00, 59.33it/s]\n", - "INFO:rasa_core.test:Finished collecting predictions.\n", - "INFO:rasa_core.test:Evaluation Results on CONVERSATION level:\n", - "INFO:rasa_core.test:\tCorrect: 62 / 62\n", - "INFO:rasa_core.test:\tF1-Score: 1.000\n", - "INFO:rasa_core.test:\tPrecision: 1.000\n", - "INFO:rasa_core.test:\tAccuracy: 1.000\n", - "INFO:rasa_core.test:\tIn-data fraction: 1\n", - "INFO:rasa_core.test:Evaluation Results on ACTION level:\n", - "INFO:rasa_core.test:\tCorrect: 232 / 232\n", - "INFO:rasa_core.test:\tF1-Score: 1.000\n", - "INFO:rasa_core.test:\tPrecision: 1.000\n", - "INFO:rasa_core.test:\tAccuracy: 1.000\n", - "INFO:rasa_core.test:\tIn-data fraction: 1\n", - "INFO:rasa_core.test:\tClassification report: \n", + "100%|███████████████████████████████████████████| 64/64 [00:01<00:00, 48.47it/s]\n", + "2019-10-23 01:04:38 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.core.test\u001b[0m - Finished collecting predictions.\n", + "2019-10-23 01:04:38 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.core.test\u001b[0m - Evaluation Results on CONVERSATION level:\n", + "2019-10-23 01:04:38 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.core.test\u001b[0m - \tCorrect: 64 / 64\n", + "2019-10-23 01:04:38 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.core.test\u001b[0m - \tF1-Score: 1.000\n", + "2019-10-23 01:04:38 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.core.test\u001b[0m - \tPrecision: 1.000\n", + "2019-10-23 01:04:38 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.core.test\u001b[0m - \tAccuracy: 1.000\n", + "2019-10-23 01:04:38 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.core.test\u001b[0m - \tIn-data fraction: 1\n", + "2019-10-23 01:04:39 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.core.test\u001b[0m - Evaluation Results on ACTION level:\n", + "2019-10-23 01:04:39 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.core.test\u001b[0m - \tCorrect: 238 / 238\n", + "2019-10-23 01:04:39 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.core.test\u001b[0m - \tF1-Score: 1.000\n", + "2019-10-23 01:04:39 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.core.test\u001b[0m - \tPrecision: 1.000\n", + "2019-10-23 01:04:39 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.core.test\u001b[0m - \tAccuracy: 1.000\n", + "2019-10-23 01:04:39 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.core.test\u001b[0m - \tIn-data fraction: 1\n", + "2019-10-23 01:04:39 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.core.test\u001b[0m - \tClassification report: \n", " precision recall f1-score support\n", "\n", - " action_listen 1.00 1.00 1.00 91\n", - " action_test 1.00 1.00 1.00 2\n", - " utter_afirmacao_botao 1.00 1.00 1.00 1\n", - " utter_bff 1.00 1.00 1.00 2\n", - " utter_botao 1.00 1.00 1.00 2\n", - " utter_comida 1.00 1.00 1.00 2\n", + " utter_filme 1.00 1.00 1.00 2\n", " utter_como_estou 1.00 1.00 1.00 2\n", - "utter_continuar_conversa 1.00 1.00 1.00 50\n", - " utter_cor 1.00 1.00 1.00 2\n", - " utter_cumprimentar 1.00 1.00 1.00 28\n", - " utter_de_onde_voce_eh 1.00 1.00 1.00 2\n", - " utter_default 1.00 1.00 1.00 1\n", - " utter_despedir 1.00 1.00 1.00 3\n", + " utter_playlist 1.00 1.00 1.00 2\n", + " utter_linguagens 1.00 1.00 1.00 2\n", + " utter_cumprimentar 1.00 1.00 1.00 29\n", " utter_diga_mais 1.00 1.00 1.00 1\n", - " utter_elogios 1.00 1.00 1.00 2\n", - " utter_esporte 1.00 1.00 1.00 2\n", - " utter_filhos 1.00 1.00 1.00 2\n", - " utter_filme 1.00 1.00 1.00 2\n", - " utter_genero 1.00 1.00 1.00 2\n", + " utter_tudo_bem 1.00 1.00 1.00 2\n", + " utter_risada 1.00 1.00 1.00 2\n", + " utter_piada 1.00 1.00 1.00 2\n", + " utter_comida 1.00 1.00 1.00 2\n", + " utter_de_onde_voce_eh 1.00 1.00 1.00 2\n", + " utter_afirmacao_botao 1.00 1.00 1.00 1\n", + " utter_onde_voce_mora 1.00 1.00 1.00 2\n", + " utter_botao 1.00 1.00 1.00 2\n", + " utter_religiao 1.00 1.00 1.00 2\n", " utter_historia 1.00 1.00 1.00 2\n", - " utter_hobby 1.00 1.00 1.00 2\n", + " utter_time 1.00 1.00 1.00 2\n", + " utter_bff 1.00 1.00 1.00 2\n", " utter_license 1.00 1.00 1.00 2\n", - " utter_linguagens 1.00 1.00 1.00 2\n", + " utter_default 1.00 1.00 1.00 1\n", + " utter_hobby 1.00 1.00 1.00 2\n", + " action_test 1.00 1.00 1.00 2\n", + " utter_o_que_sei_falar 1.00 1.00 1.00 2\n", " utter_me 1.00 1.00 1.00 2\n", + " utter_signo 1.00 1.00 1.00 2\n", + " utter_despedir 1.00 1.00 1.00 3\n", " utter_negacao_botao 1.00 1.00 1.00 1\n", - " utter_onde_voce_mora 1.00 1.00 1.00 2\n", - " utter_piada 1.00 1.00 1.00 2\n", - " utter_playlist 1.00 1.00 1.00 2\n", + " action_listen 1.00 1.00 1.00 94\n", + " utter_cor 1.00 1.00 1.00 2\n", " utter_relationship 1.00 1.00 1.00 2\n", - " utter_religiao 1.00 1.00 1.00 2\n", - " utter_risada 1.00 1.00 1.00 2\n", - " utter_signo 1.00 1.00 1.00 2\n", - " utter_star_wars 1.00 1.00 1.00 2\n", - " utter_time 1.00 1.00 1.00 2\n", + " utter_genero 1.00 1.00 1.00 2\n", + "utter_continuar_conversa 1.00 1.00 1.00 50\n", + " utter_elogios 1.00 1.00 1.00 2\n", + " utter_filhos 1.00 1.00 1.00 2\n", " utter_triste 1.00 1.00 1.00 2\n", - " utter_tudo_bem 1.00 1.00 1.00 2\n", + " utter_star_wars 1.00 1.00 1.00 2\n", + " utter_esporte 1.00 1.00 1.00 2\n", "\n", - " micro avg 1.00 1.00 1.00 232\n", - " macro avg 1.00 1.00 1.00 232\n", - " weighted avg 1.00 1.00 1.00 232\n", + " micro avg 1.00 1.00 1.00 238\n", + " macro avg 1.00 1.00 1.00 238\n", + " weighted avg 1.00 1.00 1.00 238\n", "\n", - "INFO:rasa_nlu.test:Confusion matrix, without normalization: \n", - "[[91 0 0 ... 0 0 0]\n", + "2019-10-23 01:04:39 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Confusion matrix, without normalization: \n", + "[[94 0 0 ... 0 0 0]\n", " [ 0 2 0 ... 0 0 0]\n", " [ 0 0 1 ... 0 0 0]\n", " ...\n", " [ 0 0 0 ... 2 0 0]\n", " [ 0 0 0 ... 0 2 0]\n", " [ 0 0 0 ... 0 0 2]]\n", - "INFO:rasa_core.test:Finished evaluation\n" + "2019-10-23 01:04:48 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Running model for predictions:\n", + "100%|████████████████████████████████████████| 588/588 [00:01<00:00, 313.21it/s]\n", + "2019-10-23 01:04:50 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Intent evaluation results:\n", + "2019-10-23 01:04:50 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Intent Evaluation: Only considering those 588 examples that have a defined intent out of 588 examples\n", + "2019-10-23 01:04:50 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Classification report saved to results/intent_report.json.\n", + "2019-10-23 01:04:50 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Incorrect intent predictions saved to results/intent_errors.json.\n", + "2019-10-23 01:04:51 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Confusion matrix, without normalization: \n", + "[[ 0 0 0 ... 0 0 0]\n", + " [ 0 6 0 ... 0 0 0]\n", + " [ 0 0 1 ... 0 0 0]\n", + " ...\n", + " [ 0 0 0 ... 14 0 0]\n", + " [ 0 0 0 ... 0 18 0]\n", + " [ 0 0 0 ... 0 0 23]]\n", + "2019-10-23 01:04:58 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Entity evaluation results:\n", + "2019-10-23 01:04:58 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Evaluation for entity extractor: CRFEntityExtractor \n", + "2019-10-23 01:04:58 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Classification report for 'CRFEntityExtractor' saved to 'results/CRFEntityExtractor_report.json'.\n", + "2019-10-23 01:04:58 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa.nlu.test\u001b[0m - Incorrect entity predictions saved to results/CRFEntityExtractor_errors.json.\n" ] } ], "source": [ - "!python -m rasa_core.evaluate --core $COACH_MODELS_DIALOGUE_PATH -s $COACH_STORIES_PATH -o results" + "!rasa test --stories $DATA_PATH --nlu $DATA_PATH" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "* Caso algum problme seja encotrado ele será descrito no arquivo `failed_stories.md`" + "O resultado do comando pode ser visto na pasta `results/`:" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CRFEntityExtractor_errors.json\tfailed_stories.md intent_report.json\r\n", + "CRFEntityExtractor_report.json\thist.png\t story_confmat.pdf\r\n", + "confmat.png\t\t\tintent_errors.json\r\n" + ] + } + ], + "source": [ + "!ls results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Caso algum problme seja encotrado ele será descrito no arquivo `failed_stories.md`" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -556,13 +351,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "* Outro arquivo gerado é o `story_confmat.pdf` uma matriz de confusão onde é possível visualizar a relação entre as `utters` (mensagens enviadas .\n", - "`Para visualizar a matriz de confusão neste notebook, abra-o com firefox`." + "* Outro arquivo gerado é o `story_confmat.pdf` uma matriz de confusão onde é possível visualizar a relação entre as `utters` (mensagens enviadas ." ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": { "scrolled": true }, @@ -574,24 +368,23 @@ " \n", " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 7, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "from IPython.display import IFrame\n", - "IFrame(\"results/story_confmat.pdf\", width=900, height=900)" + "IFrame(\"./results/story_confmat.pdf\", width=900, height=900)" ] }, { diff --git a/requirements.txt b/requirements.txt index ffacb926..3e0ac3bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,4 @@ -rasa==1.3.7 +rasa==1.4.2 nltk==3.4.5 elasticsearch==7.0.4 flake8==3.7.8 - -numpy==1.16.3