next_word_prediction_en_bsd.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "next-word-prediction-en-bsd.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "u3fDEotrkr5A"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Next Word Prediction w/ LSTM - [BSD Corpus](https://github.com/tsuruoka-lab/BSD) (English only)\n",
        "\n",
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1jihTIp3NQy58nkHPc6gf0yYIpW24KPxd?usp=sharing)"
      ],
      "metadata": {
        "id": "pFsCLi3Ojrq0"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Data preparation"
      ],
      "metadata": {
        "id": "Of2Kr8R2jzMT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import heapq\n",
        "import matplotlib.pyplot as plt\n",
        "from nltk.tokenize import RegexpTokenizer\n",
        "from nltk.tokenize import word_tokenize\n",
        "from keras.models import Sequential, load_model\n",
        "from keras.layers.core import Dense, Activation\n",
        "from keras.layers import LSTM\n",
        "import pickle\n",
        "from tensorflow.keras.optimizers import RMSprop"
      ],
      "metadata": {
        "id": "04wcA-OmX06I"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Yc2M5vqJsQXX",
        "outputId": "ffb6abb7-44eb-440f-8a6d-b8998db3486d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ma-3O2dwioJF",
        "outputId": "9680608d-3f49-4752-8b73-a8b16b32f699"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['phone call' 'meeting' 'face-to-face conversation' 'general chatting'\n",
            " 'presentation' 'training']\n"
          ]
        }
      ],
      "source": [
        "# Import the data set, and get all sentences\n",
        "import pandas as pd\n",
        "\n",
        "df = pd.read_json('/content/drive/MyDrive/Colab Notebooks/nwp-en-bsd/train.json')\n",
        "print(df['tag'].unique())"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "cahzn9YQqfJx",
        "outputId": "af50412f-d3c8-4130-fc19-a43524ad30cd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-d7aaf182-1b40-49e4-bd37-042e678b7072\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>tag</th>\n",
              "      <th>title</th>\n",
              "      <th>original_language</th>\n",
              "      <th>conversation</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>190329_J07_03</td>\n",
              "      <td>phone call</td>\n",
              "      <td>伝言への折り返し電話　打ち合わせ日程調整</td>\n",
              "      <td>ja</td>\n",
              "      <td>[{'no': 1, 'ja_speaker': '土井さん', 'en_speaker':...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>190329_J06_11</td>\n",
              "      <td>meeting</td>\n",
              "      <td>株主総会に関する役員向け説明（２）</td>\n",
              "      <td>ja</td>\n",
              "      <td>[{'no': 1, 'ja_speaker': '高田さん', 'en_speaker':...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>190329_J06_10</td>\n",
              "      <td>meeting</td>\n",
              "      <td>株主総会に関する役員向け説明（１）</td>\n",
              "      <td>ja</td>\n",
              "      <td>[{'no': 1, 'ja_speaker': '友井さん', 'en_speaker':...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>190329_J06_09</td>\n",
              "      <td>face-to-face conversation</td>\n",
              "      <td>信託銀行から株主名簿を受領</td>\n",
              "      <td>ja</td>\n",
              "      <td>[{'no': 1, 'ja_speaker': '高田さん', 'en_speaker':...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>190329_J06_08</td>\n",
              "      <td>face-to-face conversation</td>\n",
              "      <td>株主総会への臨場要請</td>\n",
              "      <td>ja</td>\n",
              "      <td>[{'no': 1, 'ja_speaker': '高田さん', 'en_speaker':...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>665</th>\n",
              "      <td>190315_E001_05</td>\n",
              "      <td>presentation</td>\n",
              "      <td>Presentation: Company introduction</td>\n",
              "      <td>en</td>\n",
              "      <td>[{'no': 1, 'en_speaker': 'Mr. Asano Yuki', 'ja...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>666</th>\n",
              "      <td>190315_E001_04</td>\n",
              "      <td>training</td>\n",
              "      <td>Training: training a new employee</td>\n",
              "      <td>en</td>\n",
              "      <td>[{'no': 1, 'en_speaker': 'Mr. Ben Sherman', 'j...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>667</th>\n",
              "      <td>190315_E001_03</td>\n",
              "      <td>general chatting</td>\n",
              "      <td>General chatting: Company Christmas party</td>\n",
              "      <td>en</td>\n",
              "      <td>[{'no': 1, 'en_speaker': 'Ms. Amy Lin', 'ja_sp...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>668</th>\n",
              "      <td>190315_E001_02</td>\n",
              "      <td>phone call</td>\n",
              "      <td>Phone call: requesting a lunch meeting.</td>\n",
              "      <td>en</td>\n",
              "      <td>[{'no': 1, 'en_speaker': 'Mr. John Smith', 'ja...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>669</th>\n",
              "      <td>190315_E001_01</td>\n",
              "      <td>meeting</td>\n",
              "      <td>Meeting: Wrap-up meeting (after meeting a star...</td>\n",
              "      <td>en</td>\n",
              "      <td>[{'no': 1, 'en_speaker': 'Mr. Bob Cook', 'ja_s...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>670 rows × 5 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d7aaf182-1b40-49e4-bd37-042e678b7072')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d7aaf182-1b40-49e4-bd37-042e678b7072 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d7aaf182-1b40-49e4-bd37-042e678b7072');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                 id  ...                                       conversation\n",
              "0     190329_J07_03  ...  [{'no': 1, 'ja_speaker': '土井さん', 'en_speaker':...\n",
              "1     190329_J06_11  ...  [{'no': 1, 'ja_speaker': '高田さん', 'en_speaker':...\n",
              "2     190329_J06_10  ...  [{'no': 1, 'ja_speaker': '友井さん', 'en_speaker':...\n",
              "3     190329_J06_09  ...  [{'no': 1, 'ja_speaker': '高田さん', 'en_speaker':...\n",
              "4     190329_J06_08  ...  [{'no': 1, 'ja_speaker': '高田さん', 'en_speaker':...\n",
              "..              ...  ...                                                ...\n",
              "665  190315_E001_05  ...  [{'no': 1, 'en_speaker': 'Mr. Asano Yuki', 'ja...\n",
              "666  190315_E001_04  ...  [{'no': 1, 'en_speaker': 'Mr. Ben Sherman', 'j...\n",
              "667  190315_E001_03  ...  [{'no': 1, 'en_speaker': 'Ms. Amy Lin', 'ja_sp...\n",
              "668  190315_E001_02  ...  [{'no': 1, 'en_speaker': 'Mr. John Smith', 'ja...\n",
              "669  190315_E001_01  ...  [{'no': 1, 'en_speaker': 'Mr. Bob Cook', 'ja_s...\n",
              "\n",
              "[670 rows x 5 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_convs = pd.concat([pd.json_normalize(df['conversation'][i]) for i in range(len(df))],ignore_index=True)\n",
        "trainen = df_convs[\"en_sentence\"].values.tolist()\n",
        "trainja = df_convs[\"ja_sentence\"].values.tolist()"
      ],
      "metadata": {
        "id": "wIgqXgN5kzII"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_convs"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "Oz4TefjJqpL7",
        "outputId": "6955d67b-946f-4e12-97a6-d4d64cf32b8a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-1a8fc405-7632-4496-bd6a-d195af003930\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>no</th>\n",
              "      <th>ja_speaker</th>\n",
              "      <th>en_speaker</th>\n",
              "      <th>ja_sentence</th>\n",
              "      <th>en_sentence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>土井さん</td>\n",
              "      <td>Doi-san</td>\n",
              "      <td>はい、K社システム開発部です。</td>\n",
              "      <td>Hi this is the systems development department ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>高市さん</td>\n",
              "      <td>Takaichi-san</td>\n",
              "      <td>H社の高市と申します。</td>\n",
              "      <td>My name is Takaichi from Company H.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>高市さん</td>\n",
              "      <td>Takaichi-san</td>\n",
              "      <td>いつもお世話になっております。</td>\n",
              "      <td>Thank you as always.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>土井さん</td>\n",
              "      <td>Doi-san</td>\n",
              "      <td>こちらこそ、お世話になっております。</td>\n",
              "      <td>Thank you as always as well.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>高市さん</td>\n",
              "      <td>Takaichi-san</td>\n",
              "      <td>稲田さんはいらっしゃいますか？</td>\n",
              "      <td>Is Inada-san there?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19995</th>\n",
              "      <td>28</td>\n",
              "      <td>ボブ クックさん</td>\n",
              "      <td>Mr. Bob Cook</td>\n",
              "      <td>今日の5時までに書き上げます。</td>\n",
              "      <td>I'll finish it by 5 o'clock today.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19996</th>\n",
              "      <td>29</td>\n",
              "      <td>アル ジョンソンさん</td>\n",
              "      <td>Mr. Al Johnson</td>\n",
              "      <td>完璧だね。</td>\n",
              "      <td>Perfect.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19997</th>\n",
              "      <td>30</td>\n",
              "      <td>アル ジョンソンさん</td>\n",
              "      <td>Mr. Al Johnson</td>\n",
              "      <td>明日ゼネラルマネージャーにA社の件を話しておきますね。</td>\n",
              "      <td>I'll go talk to the general manager tomorrow t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19998</th>\n",
              "      <td>31</td>\n",
              "      <td>アル ジョンソンさん</td>\n",
              "      <td>Mr. Al Johnson</td>\n",
              "      <td>彼がA社を気に入ってくれて、投資できる方向に決まることを願いましょう。</td>\n",
              "      <td>Hopefully he will like Company A and make the ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19999</th>\n",
              "      <td>32</td>\n",
              "      <td>ボブ クックさん</td>\n",
              "      <td>Mr. Bob Cook</td>\n",
              "      <td>そうですね。</td>\n",
              "      <td>I hope so too.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>20000 rows × 5 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1a8fc405-7632-4496-bd6a-d195af003930')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1a8fc405-7632-4496-bd6a-d195af003930 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1a8fc405-7632-4496-bd6a-d195af003930');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "       no  ...                                        en_sentence\n",
              "0       1  ...  Hi this is the systems development department ...\n",
              "1       2  ...                My name is Takaichi from Company H.\n",
              "2       3  ...                               Thank you as always.\n",
              "3       4  ...                       Thank you as always as well.\n",
              "4       5  ...                                Is Inada-san there?\n",
              "...    ..  ...                                                ...\n",
              "19995  28  ...                 I'll finish it by 5 o'clock today.\n",
              "19996  29  ...                                           Perfect.\n",
              "19997  30  ...  I'll go talk to the general manager tomorrow t...\n",
              "19998  31  ...  Hopefully he will like Company A and make the ...\n",
              "19999  32  ...                                     I hope so too.\n",
              "\n",
              "[20000 rows x 5 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(trainen)"
      ],
      "metadata": {
        "id": "X9aOspbbk0Gs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5819843a-eeb6-47ae-f271-f1dbd47e2c64"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "20000"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainen[140:160]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bpyu8EgEcxiq",
        "outputId": "9d1926a1-5e5a-4795-8d34-3383b9339669"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['I hope so.',\n",
              " 'Okay, anyways, we will send 2 police officers from our side to the general meeting of shareholders on the day.',\n",
              " 'Thank you very much.',\n",
              " 'I will have them speak to your reception around 9:30 in the morning.',\n",
              " 'Okay.',\n",
              " 'I will be stationed at the reception ten minutes before the start of the general meeting so please let me know.',\n",
              " 'Okay, got it.',\n",
              " 'Alright, then I look forward to working with you on the day.',\n",
              " 'Thank you for calling, this is the Judicial Scrivener office T.',\n",
              " 'Thank you as always, my name is Takada from Company I.',\n",
              " 'Thank you.',\n",
              " 'Sorry but is Mr. Takanashi in today?',\n",
              " 'Please wait a moment.',\n",
              " 'Thank you for holding, this is Takanashi.',\n",
              " 'Thank you as always, this is Takada.',\n",
              " \"I'm calling today in regards to a registration request.\",\n",
              " 'I exercised the stock option last month and the stock increased.',\n",
              " \"And I'd like to ask you to do the registration process for changing the total number of stock issued.\",\n",
              " 'Okay.',\n",
              " 'I will send you an email on the details of the number of stock.']"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#concat all english messages\n",
        "text = ' '.join(trainen[0:10000]) # Out of RAM if we use all the data\n",
        "print('length of the corpus is: :', len(text))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MNS6pWZXpnt8",
        "outputId": "c110e472-ff68-4dc1-92da-62c109f3a4f5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "length of the corpus is: : 532978\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# tokenizer\n",
        "tokenizer = RegexpTokenizer(r'\\w+')\n",
        "words = tokenizer.tokenize(text)"
      ],
      "metadata": {
        "id": "UdWEMJCYtvG0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# get unique words\n",
        "unique_words = np.unique(words)\n",
        "unique_word_index = dict((c, i) for i, c in enumerate(unique_words))"
      ],
      "metadata": {
        "id": "4lnZnbU3Yngi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(unique_words)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lExPYkbfYq5v",
        "outputId": "32778843-75cc-40af-950f-60d670f83b84"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "6197"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "unique_words[1000:1020]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k0sfpTTakRX0",
        "outputId": "c6c79997-bb9b-4fea-acb4-77384e73ee67"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['Shanghai', 'She', 'Shibata', 'Shibuya', 'Shiota', 'Shiraishi',\n",
              "       'Shiroki', 'Shirota', 'Shizuo', 'Shodoshima', 'Shop', 'Should',\n",
              "       'Shouldn', 'Shuji', 'Silence', 'Silicone', 'Similar', 'Since',\n",
              "       'Sir', 'Six'], dtype='<U16')"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Feature Engineering\n",
        "- Prepare the model input (x) and output (y)\n",
        "- Prepare word2id and id2word dictionaries"
      ],
      "metadata": {
        "id": "T3scyhWpkcty"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# model input = prev_words\n",
        "# model output = next_words\n",
        "\n",
        "WORD_LENGTH = 5\n",
        "next_words = []\n",
        "prev_words = []\n",
        "for j in range(len(words) - WORD_LENGTH):\n",
        "     prev_words.append(words[j:j + WORD_LENGTH])\n",
        "     next_words.append(words[j + WORD_LENGTH])\n",
        "print(prev_words[0])\n",
        "print(next_words[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Wh0Gzt5pY6cZ",
        "outputId": "8d3bb886-bd7a-4038-9f76-bada46cf0e2a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['Hi', 'this', 'is', 'the', 'systems']\n",
            "development\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# convert features into numpy arrays\n",
        "X = np.zeros((len(prev_words), WORD_LENGTH, len(unique_words)), dtype=bool)\n",
        "Y = np.zeros((len(next_words), len(unique_words)), dtype=bool)\n",
        "# prepare word2id index\n",
        "for i, each_words in enumerate(prev_words):\n",
        "   for j, each_word in enumerate(each_words):\n",
        "        X[i, j, unique_word_index[each_word]] = 1\n",
        "   Y[i, unique_word_index[next_words[i]]] = 1\n",
        "word2id = unique_word_index"
      ],
      "metadata": {
        "id": "ALCqO8GjZy4k"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# prepare id2word index ==> for inference\n",
        "id2word = {y:x for x,y in unique_word_index.items()}"
      ],
      "metadata": {
        "id": "mFk-haqrdNGI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Training"
      ],
      "metadata": {
        "id": "u3fDEotrkr5A"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# build model\n",
        "model = Sequential()\n",
        "model.add(LSTM(128, input_shape=(WORD_LENGTH, len(unique_words))))\n",
        "model.add(Dense(len(unique_words)))\n",
        "model.add(Activation('softmax'))"
      ],
      "metadata": {
        "id": "uRjvPHSOZ6Wj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# train model\n",
        "optimizer = RMSprop(learning_rate=0.01)\n",
        "model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n",
        "history = model.fit(X, Y, validation_split=0.05, batch_size=128, epochs=2, shuffle=True).history"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TFsRvxR8acv8",
        "outputId": "30b13ef8-c0cb-4fb4-8387-96ff7823e9ba"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/2\n",
            "769/769 [==============================] - 14s 12ms/step - loss: 5.8235 - accuracy: 0.1267 - val_loss: 5.5696 - val_accuracy: 0.1683\n",
            "Epoch 2/2\n",
            "769/769 [==============================] - 10s 13ms/step - loss: 5.4024 - accuracy: 0.1815 - val_loss: 5.9452 - val_accuracy: 0.1830\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# save model\n",
        "model.save('next_word_model.h5')\n",
        "pickle.dump(history, open(\"history.p\", \"wb\"))"
      ],
      "metadata": {
        "id": "WZlNoGh9atZb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "history"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kuFdNlQEblgJ",
        "outputId": "c1dae581-7671-4883-ea8b-66a9c0e546dd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': [0.12666530907154083, 0.1814603954553604],\n",
              " 'loss': [5.823547840118408, 5.4024271965026855],\n",
              " 'val_accuracy': [0.16827666759490967, 0.18295980989933014],\n",
              " 'val_loss': [5.569575786590576, 5.945195198059082]}"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# evaluate model\n",
        "plt.plot(history['accuracy'])\n",
        "plt.plot(history['val_accuracy'])\n",
        "plt.title('model accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'test'], loc='upper left')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        },
        "id": "-0mmtJc6bfFJ",
        "outputId": "5bcb871d-9d01-4c25-f146-150bb22ed4d8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f691df77d90>"
            ]
          },
          "metadata": {},
          "execution_count": 18
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deVyVdfr/8dcFsikqCO6I4o5KbqhZWbY42WKaLdNi5WzW9O03Nd9qvmr73sxU0zTTvk97U2qWpmnZbiWVCQKKO+CCoiggIHCu3x/3MY6IekwOh3PO9Xw8enTOue/7nOtGPW/u+3N/rltUFWOMMaa+MH8XYIwxpnmygDDGGNMgCwhjjDENsoAwxhjTIAsIY4wxDbKAMMYY0yALCGMAEXlJRO71ct0NInKGr2syxt8sIIwxxjTIAsKYICIiLfxdgwkeFhAmYLhP7dwsIitEpFxEnheRjiLyoYiUishiEYn3WP88EVkpIiUi8qmIpHosGyoiP7i3ewuIrvdZ54rIcve2X4vIcV7WeI6I/Cgie0QkX0TurLf8JPf7lbiXT3W/HiMiD4vIRhHZLSJful8bKyIFDfwcznA/vlNE3hGRV0VkDzBVREaKyFL3Z2wRkX+LSKTH9gNFZJGI7BSRbSIyU0Q6icheEUnwWG+YiGwXkQhv9t0EHwsIE2guAMYBfYEJwIfATKA9zt/nPwGISF/gDeAG97L5wPsiEun+spwDvAK0A/7rfl/c2w4FXgCuBhKAp4G5IhLlRX3lwJVAHHAO8EcRmeR+3+7uev/lrmkIsNy93UPAcOAEd01/AVxe/kwmAu+4P/M1oBb4M5AIjAZOB65119AaWAwsALoAvYGPVXUr8Clwscf7XgG8qarVXtZhgowFhAk0/1LVbapaCHwBfKuqP6pqJTAbGOpe79fAPFVd5P6CewiIwfkCPh6IAB5V1WpVfQdY5vEZ04CnVfVbVa1V1ZeBKvd2h6Wqn6pqpqq6VHUFTkid4l58GbBYVd9wf26xqi4XkTDgt8D1qlro/syvVbXKy5/JUlWd4/7MClX9XlW/UdUaVd2AE3D7azgX2KqqD6tqpaqWquq37mUvA1MARCQcuBQnRE2IsoAwgWabx+OKBp7Huh93ATbuX6CqLiAf6OpeVqgHdqrc6PG4O3Cj+xRNiYiUAN3c2x2WiIwSkSXuUzO7gWtwfpPH/R5rG9gsEecUV0PLvJFfr4a+IvKBiGx1n3a634saAN4DBohICs5R2m5V/e4X1mSCgAWECVabcb7oARARwflyLAS2AF3dr+2X7PE4H7hPVeM8/mupqm948bmvA3OBbqraFngK2P85+UCvBrbZAVQeYlk50NJjP8JxTk95qt+S+UkgF+ijqm1wTsF51tCzocLdR2Fv4xxFXIEdPYQ8CwgTrN4GzhGR092DrDfinCb6GlgK1AB/EpEIEZkMjPTY9lngGvfRgIhIK/fgc2svPrc1sFNVK0VkJM5ppf1eA84QkYtFpIWIJIjIEPfRzQvAIyLSRUTCRWS0e8xjNRDt/vwI4FbgSGMhrYE9QJmI9Af+6LHsA6CziNwgIlEi0lpERnks/w8wFTgPC4iQZwFhgpKqrsL5TfhfOL+hTwAmqOo+Vd0HTMb5ItyJM14xy2PbDOAPwL+BXcAa97reuBa4W0RKgdtxgmr/+24CzsYJq504A9SD3YtvAjJxxkJ2An8FwlR1t/s9n8M5+ikHDriqqQE34QRTKU7YveVRQynO6aMJwFYgDzjVY/lXOIPjP6iq52k3E4LEbhhkjPEkIp8Ar6vqc/6uxfiXBYQx5mciMgJYhDOGUurveox/2SkmYwwAIvIyzhyJGywcDNgRhDHGmEOwIwhjjDENCprGXomJidqjRw9/l2GMMQHl+++/36Gq9efWAEEUED169CAjI8PfZRhjTEARkUNezmynmIwxxjTIAsIYY0yDLCCMMcY0KGjGIBpSXV1NQUEBlZWV/i7F56Kjo0lKSiIiwu7tYoxpHEEdEAUFBbRu3ZoePXpwYOPO4KKqFBcXU1BQQEpKir/LMcYEiaA+xVRZWUlCQkJQhwOAiJCQkBASR0rGmKYT1AEBBH047Bcq+2mMaTpBfYrJGGOCkiqUFcH2HCjKhRaRkP7bRv8YCwgfKykp4fXXX+faa689qu3OPvtsXn/9deLi4nxUmTEmIOzdCUXZUJQD23Od/xflQMXOunWSRlhABKKSkhKeeOKJgwKipqaGFi0O/eOfP3++r0szxjQnFSV1AbA91x0KuVBeVLdOVBto3x9SJ0CHVOdxhwEQ28EnJVlA+Nj06dNZu3YtQ4YMISIigujoaOLj48nNzWX16tVMmjSJ/Px8Kisruf7665k2bRpQ1zqkrKyMs846i5NOOomvv/6arl278t577xETE+PnPTPG/CJVZbB9lRMAnkcEpZvr1oloBe37QZ9x7iBIhQ79oU1XaMLxxpAJiLveX0n25j2N+p4DurThjgkDD7vOgw8+SFZWFsuXL+fTTz/lnHPOISsr6+fLUV944QXatWtHRUUFI0aM4IILLiAhIeGA98jLy+ONN97g2Wef5eKLL+bdd99lypQpjbovxphGVl3hBIHn0UBRDuzeVLdOi2hI7AspYw4MgrbJEOb/a4hCJiCai5EjRx4wV+Gxxx5j9uzZAOTn55OXl3dQQKSkpDBkyBAAhg8fzoYNG5qsXmPMEdRUwY68A4Ngew7sXA+477cTFuEEQbcRMOxKJww6pEJ8DwgLP6aP37q7km17KhncrfHHK0MmII70m35TadWq1c+PP/30UxYvXszSpUtp2bIlY8eObXAuQ1RU1M+Pw8PDqaioaJJajTEeaquheG3dlUP7TxEVrwWtddaRcEjoBZ3SIO1i52igwwBo1xPCG7fLgculvPbtRv66YBWd2kbz0Q0nExbWuKefQiYg/KV169aUljZ898bdu3cTHx9Py5Ytyc3N5Ztvvmni6owxB3HVOr/91w+CHXngqnavJNAuxfnyTz2v7oggoTe0iDrs2zeGVVtLmTFrBT9sKuGk3oncd/6gRg8H8HFAiMh44J9AOPCcqj5Yb/nJwKPAccAlqvqOx7K/AefgTOZbBFyvAXh/1ISEBE488UQGDRpETEwMHTt2/HnZ+PHjeeqpp0hNTaVfv34cf/zxfqzUmBDjcjnjAfsHifefItqRBzUeR/Jxyc7YQJ9f1QVBYl+IaPoLRSqra/nXJ3k8/dk62sRE8MjFgzl/aFefTZT12T2pRSQcWA2MAwqAZcClqprtsU4PoA1wEzB3f0CIyAnA34GT3at+CcxQ1U8P9Xnp6ela/4ZBOTk5pKamNs4OBYBQ219jvKIKewoPPBooyobtq6G6vG69Nl3dl42m1g0Yt+8HUbH+q93D12t2MHN2JhuK93LBsCRuOSeVdq0ij/l9ReR7VU1vaJkvjyBGAmtUdZ27iDeBicDPAaGqG9zLXPW2VSAaiAQEiAC2+bBWY0ygU4WybR5HBO5TRNtzocrjCsbYjk4QDLviwCCIaZ6TUneV7+O++Tm8830B3RNa8trvR3Fi78Qm+WxfBkRXIN/jeQEwypsNVXWpiCwBtuAExL9VNafxSzTGBKTyHQdPKCvKhsqSunVi2jljBMdd7HEJaSq0bOe/uo+CqjJneSH3fJDDnopqrh3biz+d3ofoiGO76uloNMtBahHpDaQCSe6XFonIGFX9ot5604BpAMnJyU1bpDHG9yp21V026jlWUL69bp2ots4X/8BJdSHQIRVatW/SSWWNaVPxXm6Zk8kXeTsY0i2OBy9Io3+nNk1ehy8DohDo5vE8yf2aN84HvlHVMgAR+RAYDRwQEKr6DPAMOGMQx1qwMcZPKve4J5XVu3KodEvdOpGxzqmgvmceGAStOwdsENRXXevi+S/X8+ji1bQIC+PuiQO5fFR3wn1whZI3fBkQy4A+IpKCEwyXAJd5ue0m4A8i8gDOKaZTcK52MsYEsn17YcequqOB/UcEuz3ORreIgfZ9oefYAweN2yQ1i9nFvvJTfgnTZ2WSs2UPvxrQkbsmDqRzW/+21PFZQKhqjYhcByzEucz1BVVdKSJ3AxmqOldERgCzgXhggojcpaoDgXeA04BMnAHrBar6vq9qNcY0supKKM47+BLSXRv5eXZxeKR7dvEoGD61rvlcI8wuDiRlVTU8tHAVLy/dQIfWUTw1ZTjjB3Xyd1mAj8cgVHU+ML/ea7d7PF5G3TiD5zq1wNW+rK2p/NJ23wCPPvoo06ZNo2XLlj6ozJhGULMPdq49sMVEUQ7sXAfqvjgxrIUzgazzEBh8mTO7uH2qe3ZxsxwGbTKLs7dx23tZbN1TyRXHd+emM/vRJrr53Fc+tP90msCh2n1749FHH2XKlCkWEMb/amtg1/qDLyEtzgNXjbOOhDlf+u37w8Dz664cSujt3NDG/KxoTyV3vr+S+Zlb6dexNf++bBjDu8f7u6yDWED4mGe773HjxtGhQwfefvttqqqqOP/887nrrrsoLy/n4osvpqCggNraWm677Ta2bdvG5s2bOfXUU0lMTGTJkiX+3hUTClwuKNlQ78qhXNixGmqr6taL6+5cQtpvvPP/9v3ds4uj/VZ6IHC5lDeWbeLBD3OpqnFx85n9+MOYnkS2aJ5jK6ETEB9Oh62ZjfuendLgrAcPu4pnu++PPvqId955h++++w5V5bzzzuPzzz9n+/btdOnShXnz5gFOj6a2bdvyyCOPsGTJEhITm2ZSjAkhqs7AcP1LSHeshuq9deu1SXKOBHqNrbtyqH0/iGx1yLc2DcvbVsqMWZlkbNzF6J4J3D85jZTE5v1zDJ2AaAY++ugjPvroI4YOHQpAWVkZeXl5jBkzhhtvvJH/+7//49xzz2XMmDF+rtQEDVUo3Xpgi4miXOeS0n0eTSRjOzljA8On1t2lrH0/iG76a++DTWV1LU8sWcOTn62lVVQL/n7hcVw4PMln/ZMaU+gExBF+028KqsqMGTO4+uqDx99/+OEH5s+fz6233srpp5/O7bff3sA7GHMYZdsPvkvZ9hyo3F23TstE5yhgyKV1l5C27x8ws4sDzTfripk5K5N1O8o5f2hXbj0nlYRY33d7bSyhExB+4tnu+8wzz+S2227j8ssvJzY2lsLCQiIiIqipqaFdu3ZMmTKFuLg4nnvuuQO2tVNM5gB7d9Y7GnA/3ltct050nHt28WTnaGD/lUOx7f1Xdwgp2buPB+bn8lZGPt3axfCf347k5L6B97O3gPAxz3bfZ511FpdddhmjR48GIDY2lldffZU1a9Zw8803ExYWRkREBE8++SQA06ZNY/z48XTp0sUGqUNR5e66exd7jhWUefStjGztfPn3O/vAIGjdKWhmFwcSVeX9FVu4+/2V7NpbzdWn9OSG0/sSExmY8zp81u67qVm779Db36Cxr9x9FODZjjoX9hTUrRPR0hkT2H/P4v1XDrVNsiBoJvJ37uXWOVl8tno7g5Pacv/kNAZ2aevvso7IX+2+jTGeqiucq4QOCIIcKNlYt054lHO5aPcTDgyCuO5B3WYikNXUunjxqw08smg1InDHhAFcObqH3/onNSYLCGMaW82+ujYTngPGu9bXm13cB7oOh6FT6q4ciu8R8rOLA0lmwW6mz1rBys17OL1/B+6eNIiucf7tn9SYgv5voqoGxOVkxypYThUGlNoap6VE/SuHdq71mF0c7swu7jgA0i6sC4KEXo1+E3vTdMqranhk0Wpe/Go9CbFRPHH5MM4a1CnovmuCOiCio6MpLi4mISEh6P7gPKkqxcXFREfbLFafcNXCrg0HtpgoynGOEmr3uVcS57f/DqmQem7dWEFCH5tdHGSW5BZx65wsCksquHxUMn8Z35+2McEZ9kEdEElJSRQUFLB9+/YjrxzgoqOjSUo6qO+hORoulzO7+IBLSHOcK4k8b2LfNtn58u99et2VQ4n9INJ6ZgWzotJK7n4/mw9WbKF3h1j+e81oRvQI7vkjQR0QERERpKSk+LsM09yowp7NBx4N7H/seRP71p2dI4L033ncyL4fRLX2X+2myblcytsZ+dw/P4fKahf/O64vV5/Sk6gWgXnp6tEI6oAwIU4VyooOvktZUS5UecwubtXe+fIfOsXjyqF+ENP8umuaprWmqIyZszL5bsNORqW04/7JafRqH+vvspqMBYQJDuXFdRPJPAeMK3bWrRMT73z5p13ocUSQCq0S/Fe3aZaqamp58tO1PLFkLTGR4fztguO4KD0w+ic1JgsIE1gqSuoCwHOsoLyobp2oNs7VQqkT6noNdRgAsR1sUpk5ou/W72TGrBWs3V7OeYO7cNu5A2jfOnD6JzUmCwjTPFWV1bWZ8DwiKN1ct05EK+dUUJ9xdUcDHVKhTRcLAnPUdu+t5sEFObzxXT5J8TG89JsRjO3Xwd9l+ZUFhPGv6gonCDyPBopyYPemunVaRDuzi1PGeARBf+dqIptdbI6RqjIvcwt3zs1mZ3kV007uyQ1n9KFlpH092k/ANI2aKtiRd/AlpDvX8/NN7MMi3DexHwHDr6w7Igixm9ibplOway+3v7eST3KLSOvalpd+M4JBXZt//6SmYgFhGldtNRSvPfAuZdtznde01llHwp37FHdKg7SL6waM2/W02cWmSdS6lJe+3sDDH61CFW49J5WpJ/SgRbgdkXqygDC/jKvW+e2//iWkO/LAVe1eSaBdijNAnHpeXRAk9IYWoTnoZ/wvq3A3M2Zlklm4m1P7teeeSYNIirdJjg2xgDCH53I54wGeRwNF2U4QeM4ujkt2gqDPr+qCILEvRARP4zIT2Pbuq+HRxXk8/+V64ltG8u/LhnJOWueQu3T1aFhAGIcq7Ck8OAi2rz5wdnGbrs5loymn1A0Yt+8HUaEzecgEnk9XOf2TCnZVcOnIbkwfn0rblnY680gsIEKNqnNHMs97Fhe5+w1V7albL7ajEwTDrjgwCGLi/Fe7MUdpR1kV93yQzXvLN9OrfSvevno0I1OCu39SY7KACGblO+oFgfuooLKkbp2Yds6poeMuPnAugd3E3gQwVeW/3xdw37wcKvbVcsMZffjj2F4h0T+pMVlABIOKXQfes3j/KaJyjy62UW3dN7GfVBcCHVKdPkR2DtYEkXXby5g5O5Nv1u1kZI923D95EL07WIPFX8KnASEi44F/AuHAc6r6YL3lJwOPAscBl6jqO+7XTwX+4bFqf/fyOb6st9mr3OOeVFbvyqHSLXXrRMY6p4b6nnlgELTubEFggtq+GhdPf7aWfy1ZQ1SLMB6YnMav07sRFgS3/vQXnwWEiIQDjwPjgAJgmYjMVdVsj9U2AVOBmzy3VdUlwBD3+7QD1gAf+arWZmdfucfsYo8jgt35deu0iIH2faHnWHevof1tJpJsdrEJOd9v3Mn0dzPJKyrj3OM6c/uEAXRobTdqOla+PIIYCaxR1XUAIvImMBH4OSBUdYN7mesw73Mh8KGq7vVdqX5SXVl372LPsYJdG/l5dnF4pHt28SgYPrUuCOK62+xiE/L2VFbztwW5vPrNJrrGxfDC1HRO69/R32UFDV8GRFfA41deCoBRv+B9LgEeaWiBiEwDpgEkJyf/grduIjX7nPsUe7aYKMpx7md8wE3se0PnITD4MqfXUPv9s4ttqMgYT6rKgqyt3DF3JTvKqvjdSSn877i+tIqyfyuNqVn/NEWkM5AGLGxouao+AzwDkJ6erk1YWsNqa2DX+oOvHCrO87iJfZjzpd++PwycXBcECb2hRaR/6zcmAGwuqeD291ayOGcbAzq34bmr0jkuyS6/9gVfBkQh0M3jeZL7taNxMTBbVauPuGZTcrmgZMPBdynbsRpqq+rWi+/hfPn3G+++S1l/9+xiOzdqzNGqdSmvLN3A3xeuolaVmWf357cnplj/JB/yZUAsA/qISApOMFwCXHaU73EpMKOxC/OaqjMwXP8S0h2rodpjSKRNkjMu0GtsXRC07weRrfxWujHBJGfLHqbPyuSn/BJO7tue+yYNols765/kaz4LCFWtEZHrcE4PhQMvqOpKEbkbyFDVuSIyApgNxAMTROQuVR0IICI9cI5APvNVjR7FQulWj6OB/WMFq2Bfad16sZ2cIBg+te4uZe37QXQbn5doTCiq2FfLPz/O49kv1hEXE8E/LxnCeYO7WP+kJiKq/j913xjS09M1IyPj6DfcXQhPjoZKj5vYt0z0uGexRxDY7GJjmswXedu5ZXYWm3bu5eL0JGaenUpcSxuna2wi8r2qpje0rFkPUjeJ1p0g7SJI7Fc3YBzb3t9VGROyisuquG9eDrN+LCQlsRWv/2EUJ/RK9HdZIckCIiwcznnY31UYE/JUlXd/KOS+edmUVdXwp9N6c+2pvYmOsPk+/mIBYYzxuw07ypk5O5Ov1xYzvHs8D0xOo29H65/kbxYQxhi/qa518czn63js4zwiw8O4d9IgLhuZbP2TmgkLCGOMX/ywaRcz3s1k1bZSzhrUiTvPG0jHNjZHqDmxgDDGNKnSymr+vnAVr3yzkU5tonn2ynTGDbD+Sc2RBYQxpsksXLmVO95bybbSSq4a3YObzuxHrPVParbsT8YY43Nbd1dyx9wsFq7cRv9OrXnqiuEM6Wb9k5o7CwhjjM/UupTXvt3I3xasorrWxfSz+vO7k1KIsP5JAcECwhjjE7lb9zBjViY/biphTJ9E7p00iO4J1p8skFhAGGMaVWV1Lf/6JI+nP1tHm5gI/vHrwUwa0tX6JwUgCwhjTKP5es0OZs7OZEPxXi4YlsQt56TSrpX1TwpUFhDGmGO2q3wf983P4Z3vC+iR0JLXfj+KE3tb/6RAZwFhjPnFVJU5ywu554Mc9lRU8z+n9uL/ndbH+icFCQsIY8wvsql4L7fMyeSLvB0MTY7jgclp9O9k90YJJhYQxpijUl3r4vkv1/Po4tW0CAvjnokDuWxUd8Ktf1LQsYAwxnjtp/wSps/KJGfLHs4c2JG7zhtEp7bWPylYWUAYY46orKqGhxau4uWlG+jQOoqnpgxn/KBO/i7L+JgFhDHmsBZnb+O297LYuqeSK47vzs1n9qN1dIS/yzJNwALCGNOgoj2V3Pn+SuZnbqVfx9Y8fvkwhiXH+7ss04QsIIwxB3C5lNe/28RfF+RSVePi5jP7Me3kntY/KQRZQBhjfrZ6WykzZmXy/cZdnNArgfvOTyMl0fonhSoLCGMMldW1PLFkDU9+tpZWUS146KLBXDDM+ieFOgsIY0Lc0rXF3DI7k3U7yjl/aFduPSeVhNgof5dlmgELCGNCVMnefdw/P4e3MwpIbteSV343kjF92vu7LNOMWEAYE2JUlbk/beaeD7LZtbeaa07pxfWn9yEm0vonmQP59LIEERkvIqtEZI2ITG9g+cki8oOI1IjIhfWWJYvIRyKSIyLZItLDl7UaEwryd+5l6ovLuP7N5XSNi+H9605i+ln9LRxMg7w6ghCRWcDzwIeq6vJym3DgcWAcUAAsE5G5qprtsdomYCpwUwNv8R/gPlVdJCKxgFefa4w5WE2tixe+Ws8/FuURJnDHhAFcObqH9U8yh+XtKaYngN8Aj4nIf4EXVXXVEbYZCaxR1XUAIvImMBH4OSBUdYN72QFf/iIyAGihqovc65V5Wacxpp4VBSXMmJXJys17OCO1A3dPHESXuBh/l2UCgFcBoaqLgcUi0ha41P04H3gWeFVVqxvYrCuQ7/G8ABjlZV19gRL3kUsKsBiYrqq1niuJyDRgGkBycrKXb21MaCivquGRRat58av1JMZG8eTlwxg/qJNdumq85vUgtYgkAFOAK4AfgdeAk4CrgLE+qGsMMBTnNNRbOKeinvdcSVWfAZ4BSE9P10auwZiA9UnuNm6bs5LCkgouH5XMX8b3p22M9U8yR8fbMYjZQD/gFWCCqm5xL3pLRDIOsVkh0M3jeZL7NW8UAMs9Tk/NAY6nXkAYYw5UVFrJXe9nM2/FFvp0iOWda0aT3qOdv8syAcrbI4jHVHVJQwtUNf0Q2ywD+ohICk4wXAJc5uXnLQPiRKS9qm4HTgMOFUTGhDyXS3krI58H5udQWe3ixnF9ufqUXkS2sP5J5pfzNiAGiMiPqloCICLxwKWq+sShNlDVGhG5DlgIhAMvqOpKEbkbyFDVuSIyApgNxAMTROQuVR2oqrUichPwsTgnTL/HGe8wxtSzpqiUmbOy+G7DTkaltOP+yWn0ah/r77JMEBDVI5+6F5Hlqjqk3ms/qupQn1V2lNLT0zUjww4yTOioqqnlyU/X8sSStcREhnPL2alclJ5kg9DmqIjI94c6E+TtEUS4iIi608Q9xyGysQo0xhyd79bvZMasFazdXs7EIV247dwBJFr/JNPIvA2IBTgD0k+7n1/tfs0Y04R2763mwQU5vPFdPknxMbz0mxGM7dfB32WZIOVtQPwfTij80f18EfCcTyoyxhxEVZmXuYU752aza+8+pp3ckxvO6EPLSGunZnzH24lyLuBJ93/GmCZUsGsvt7+3kk9yi0jr2paXfjOCQV3b+rssEwK8nQfRB3gAGABE739dVXv6qC5jQl6tS3np6w08/JHT1ea2cwdw1ejutLBbf5om4u3x6YvAHcA/gFNx+jLZ31JjfCSrcDczZmWSWbib0/p34O6JA0mKb+nvskyI8TYgYlT1Y/eVTBuBO0Xke+B2H9ZmTMjZu6+GRxfn8fyX64lvGcm/LxvKOWmd7dJV4xfeBkSViIQBee7Jb4WAzcQxphF9uqqIW+dkUbCrgktHJjN9fH/atrT+ScZ/vA2I64GWwJ+Ae3BOM13lq6KMCSXbS6u454Ns5v60mV7tW/H21aMZmWL9k4z/HTEg3JPifq2qNwFlOOMPxphjpKr8N6OA++bnULGvlhvO6MMfx/YiqoXd3c00D0cMCHdfpJOaohhjQsW67WXMnJ3JN+t2MrKH0z+pdwc7a2uaF29PMf0oInOB/wLl+19U1Vk+qcqYILWvxsXTn63lX0vWEN0ijAcnp3FxejfC7NafphnyNiCigWKcttv7KWABYYyXMjbsZMasTPKKyjj3uM7cPmEAHVpHH3lDY/zE25nUNu5gzC+0u6Kavy3I5bVvN9E1LoYXp47g1P7WPxXQIKUAABYSSURBVMk0f97OpH4R54jhAKr620avyJggoaosyNrKHXNXsqOsit+flMKfx/WlVZT1TzKBwdu/qR94PI4Gzgc2N345xgSHzSUV3P5eFotzihjYpQ3PXzWCtCTrn2QCi7enmN71fC4ibwBf+qQiYwJYrUv5z9INPLRwFS6FW85O5Tcn9rD+SSYg/dJj3T6AnUQ1xkP25j3MmLWCnwp2c0rf9tw7aRDd2ln/JBO4vB2DKOXAMYitOPeIMCbkVeyr5Z8f5/HsF+uIi4ngn5cM4bzBXax/kgl43p5iau3rQowJRJ+v3s4tczLJ31nBr9O7MePs/sS1tLvxmuDg7RHE+cAnqrrb/TwOGKuqc3xZnDHNVXFZFffOy2H2j4X0TGzFG384ntG9EvxdljGNytsxiDtUdfb+J6paIiJ3ABYQJqSoKu/+UMi987Ipr6rhT6f15tpTexMdYf2TTPDxNiAaugTDLuY2IWXDjnJmzs7k67XFpHeP54HJafTpaGdfTfDy9ks+Q0QeAR53P/8f4HvflGRM81Jd6+KZz9fx2Md5RIaHcd/5g7h0RLL1TzJBz9uA+H/AbcBbOFczLcIJCWOC2g+bdjHj3UxWbSvl7LRO3DFhIB3bWP8kExq8vYqpHJju41qMaTZKK6v5+8JVvPLNRjq1iebZK9MZN6Cjv8sypkl5Nb1TRBa5r1za/zxeRBZ6sd14EVklImtE5KCAEZGTReQHEakRkQvrLasVkeXu/+Z6U6cxjWHhyq2Me+RzXvlmI1NP6MGi/z3FwsGEJG9PMSWqasn+J6q6S0QOO5PafSe6x4FxQAGwTETmqmq2x2qbgKnATQ28RYWqDvGyPmOO2dbdldwxN4uFK7eR2rkNT10xnCHd4o68oTFBytuAcIlIsqpuAhCRHjTQ3bWekcAaVV3n3uZNYCLwc0Co6gb3MtdRVW1MI6p1Ka99u5G/LVhFjcvF9LP687uTUoiw/kkmxHkbELcAX4rIZ4AAY4BpR9imK5Dv8bwAGHUUtUWLSAZQAzzY0KQ8EZm2v47k5OSjeGtjHLlb9zBjViY/biphTJ9E7puURnKC9U8yBrwfpF4gIuk4X8Y/4kyQq/BlYUB3VS0UkZ7AJyKSqapr69X1DPAMQHp6+pGOaIz5WWV1Lf/6JI+nP1tHm5gIHv31ECYOsf5JxnjyttXG74HrgSRgOXA8sJQDb0FaXyHQzeN5kvs1r6hqofv/60TkU2AosPawGxnjha/W7OCW2ZlsKN7LhcOTuOXsVOJbWf8kY+rz9iTr9cAIYKOqnorzZV1y+E1YBvQRkRQRiQQuAby6Gsl9lVSU+3EicCIeYxfG/BI7y/dx49s/cflz3wLw+u9H8dBFgy0cjDkEb8cgKlW1UkQQkShVzRWRfofbQFVrROQ6YCEQDrygqitF5G4gQ1XnisgIYDYQD0wQkbtUdSCQCjztHrwOwxmDsIAwv4iqMmd5Ifd8kMOeimquO7U3151m/ZOMORJvA6LAPQ9iDrBIRHYBG4+0karOB+bXe+12j8fLcE491d/uayDNy9qMOaSNxeXcOieLL/J2MDQ5jgcnH0e/TtY/yRhveDtIfb774Z0isgRoCyzwWVXGHKPqWhfPfbGeRxevJiI8jHsmDuTyUd2tf5IxR+GoO7Kq6me+KMSYxrI8v4Tp764gd2sp4wd24s7zBtKprfVPMuZoWctuEzTKqmp4aOEqXl66gY6to3n6iuGcObCTv8syJmBZQJigsCh7G7e/l8XWPZVceXx3bjqzH62jI/xdljEBzQLCBLRteyq5c+5KPszaSr+OrXn88mEMS473d1nGBAULCBOQXC7l9e828dcPc9lX6+Iv4/vxhzE9rX+SMY3IAsIEnNXbSpkxK5PvN+7ixN4J3DcpjR6JrfxdljFBxwLCBIzK6loeX7KGpz5bS2xUCx6+aDCTh3W1/knG+IgFhAkIS9cWM3N2Jut3lDN5aFduOSeVhNgof5dlTFCzgDDNWsnefdw/P4e3MwpIbteSV343kjF92vu7LGNCggWEaZZUlbk/bebu97Mpqajmj2N78afT+hATaf2TjGkqFhCm2cnfuZdb52Tx2ertDO4WxyvnpzGgSxt/l2VMyLGAMM1GTa2LF75azz8W5REmcOeEAVwxugfh1j/JGL+wgDDNwoqCEmbMymTl5j2ckdqRuycOpEtcjL/LMiakWUAYvyqvquGRRat58av1JMZG8dSUYZw5sJNdumpMM2ABYfzmk9xt3DZnJYUlFUw5Ppm/jO9PG+ufZEyzYQFhmlxRaSV3vZ/NvBVb6NMhlneuGU16j3b+LssYU48FhGkyLpfyVkY+D8zPobLGxU2/6su0k3sR2cL6JxnTHFlAmCaxpqiUmbOy+G7DTo7v2Y77z0+jZ/tYf5dljDkMCwjjU1U1tTyxZC1PfrqWmMhw/nbhcVw0PMkGoY0JABYQxme+Xef0T1q7vZyJQ7pw27kDSLT+ScYEDAsI0+h2763mwQU5vPFdPknxMbz825Gc0tf6JxkTaCwgTKNRVT5YsYW73s9m1959XH1yT64/ow8tI+2vmTGByP7lmkZRsGsvt83JYsmq7aR1bctLvxnBoK5t/V2WMeYYWECYY1JT6+Klrzfw8EerEYHbzx3AVSdY/yRjgoEFhPnFsgp3M2NWJpmFuzmtfwfumTSIrtY/yZig4dMZSiIyXkRWicgaEZnewPKTReQHEakRkQsbWN5GRApE5N++rNMcnb37arhvXjYTH/+KLbsrefyyYTx/VbqFgzFBxmdHECISDjwOjAMKgGUiMldVsz1W2wRMBW46xNvcA3zuqxrN0VuyqohbZ2dRWFLBpSOTmT6+P21bWv8kY4KRL08xjQTWqOo6ABF5E5gI/BwQqrrBvcxVf2MRGQ50BBYA6T6s03hhe2kV93yQzdyfNtO7Qyz/vWY0I6x/kjFBzZcB0RXI93heAIzyZkMRCQMeBqYAZxxmvWnANIDk5ORfXKg5NFXl7Yx87p+fS8W+Wv58Rl+uGduTqBZ2609jgl1zHaS+FpivqgWHa8mgqs8AzwCkp6drE9UWMtZuL2PmrEy+Xb+TkSlO/6TeHax/kjGhwpcBUQh083ie5H7NG6OBMSJyLRALRIpImaoeNNBtGt++GhdPfbaWf3+yhuiIMP56QRoXDe9GmF26akxI8WVALAP6iEgKTjBcAlzmzYaqevn+xyIyFUi3cGgaGRt2MmNWJnlFZUwY3IXbzk2lQ+tof5dljPEDnwWEqtaIyHXAQiAceEFVV4rI3UCGqs4VkRHAbCAemCAid6nqQF/VZA5td0U1f1uQy2vfbqJrXAwvTh3Bqf07+LssY4wfiWpwnLpPT0/XjIwMf5cRcFSVBVlbuWPuSnaUVfHbE1P487i+tIpqrsNTxpjGJCLfq2qDV4rat0AI21xSwe3vZbE4p4hBXdvw/FUjSEuy/knGGIcFRAiqdSn/WbqBhxauwqVw6zmpTD2hBy3C7dafxpg6FhAhJnvzHmbMWsFPBbs5pW977p00iG7tWvq7LGNMM2QBESIq9tXyz4/zePaLdcS3jOCxS4cy4bjOdutPY8whWUCEgM9Xb+eWOZnk76zg1+ndmHF2f+JaRvq7LGNMM2cBEcSKy6q4d14Os38spGdiK96cdjzH90zwd1nGmABhARGEVJV3fyjk3nnZlFfV8KfT+3Dt2F5ER1j/JGOM9ywggsz6HeXcMjuTr9cWk949ngcmp9GnY2t/l2WMCUAWEEFiX42LZ79Yxz8/ziOqRRj3nT+IS0ckW/8kY8wvZgERBL7fuIuZszJZta2Uc9I6c8eEAXRoY/2TjDHHxgIigO2prObvC1bx6rcb6dwmmueuTOeMAR39XZYxJkhYQAQop39SFkWlVUw9oQc3/qofsdY/yRjTiOwbJcBs2V3BHe+t5KPsbaR2bsMzV6QzuFucv8syxgQhC4gAUetSXvt2I39bsIoal4sZZ/XntyelEGH9k4wxPmIBEQByt+5h+ruZLM8vYUyfRO6blEZygvVPMsb4lgVEM1ZZXctjH+fxzOfraBsTwaO/HsLEIV2sf5IxpklYQDRTX63ZwczZmWws3stFw5OYeXYq8a2sf5IxpulYQDQzO8v3ce+8bGb9UEiPhJa8/vtRnNA70d9lGWNCkAVEM6GqzP6xkHs+yKa0sobrTu3Ndaf1tv5Jxhi/sYBoBjYWl3PL7Cy+XLODYclxPDD5OPp1sv5Jxhj/soDwo+paF899sZ5HF68mMjyMeyYN4vKR1j/JGNM8WED4yfL8Eqa/u4LcraWMH9iJO88bSKe21j/JGNN8WEA0sbKqGh5auIqXl26gY+tonrliOL8a2MnfZRljzEEsIJrQouxt3P5eFlv3VHLV6B7c+Ku+tI6O8HdZxhjTIAuIJrBtTyV3zl3Jh1lb6d+pNU9cPoyhyfH+LssYYw7LAsKHXC7l9e828dcPc9lX6+Iv4/vxhzE9rX+SMSYg+PSbSkTGi8gqEVkjItMbWH6yiPwgIjUicqHH693dry8XkZUico0v6/SF1dtKuejppdw6J4vB3eL46M8nc+3Y3hYOxpiA4bMjCBEJBx4HxgEFwDIRmauq2R6rbQKmAjfV23wLMFpVq0QkFshyb7vZV/U2lsrqWh5fsoanPltLbFQLHrl4MOcP7Wr9k4wxAceXp5hGAmtUdR2AiLwJTAR+DghV3eBe5vLcUFX3eTyNwsdHOo1l6dpiZs7OZP2OciYP68qt5wygnfVPMsYEKF8GRFcg3+N5ATDK241FpBswD+gN3NzQ0YOITAOmASQnJx9TsceiZO8+7p+fw9sZBSS3a8mrvxvFSX2sf5IxJrA120FqVc0HjhORLsAcEXlHVbfVW+cZ4BmA9PR09UONzP1pM3e/n01JRTV/HNuLP53Wh5hI659kjAl8vgyIQqCbx/Mk92tHRVU3i0gWMAZ4p5FqO2b5O/dyy5wsPl+9ncHd4nh1chqpndv4uyxjjGk0vgyIZUAfEUnBCYZLgMu82VBEkoBiVa0QkXjgJOAfPqv0KNTUunjhq/U8smg14SLcdd5AphzfnXDrn2SMCTI+CwhVrRGR64CFQDjwgqquFJG7gQxVnSsiI4DZQDwwQUTuUtWBQCrwsIgoIMBDqprpq1q9taKghOnvZpK9ZQ9npHbk7okD6RIX4++yjDHGJ0S1yU/d+0R6erpmZGT45L3Lq2p4+KPVvPT1ehJjo7h74kDOHNjJLl01xgQ8EfleVdMbWtZsB6mbi09yt3HbnJVs3l3BlFHduXl8P9pY/yRjTAiwgDiEotJK7no/m3krttC3YyzvXDOa4d3b+bssY4xpMhYQ9bhcypvL8nngwxyqalzc9Ku+TDu5F5EtAmKunjHGNBoLCA9rikqZMSuTZRt2MbpnAvedP4ie7WP9XZYxxviFBQRQVVPLE0vW8sSna2gV1YK/X3gcFw5PskFoY0xIC/mAyN+5l6te/I5128uZNKQLt547gMTYKH+XZYwxfhfyAdGxTTQpCa24Y8JATunb3t/lGGNMsxHyARHZIoznp47wdxnGGNPs2KU5xhhjGmQBYYwxpkEWEMYYYxpkAWGMMaZBFhDGGGMaZAFhjDGmQRYQxhhjGmQBYYwxpkFBc8MgEdkObDyGt0gEdjRSOYEi1PY51PYXbJ9DxbHsc3dVbbCNRNAExLESkYxD3VUpWIXaPofa/oLtc6jw1T7bKSZjjDENsoAwxhjTIAuIOs/4uwA/CLV9DrX9BdvnUOGTfbYxCGOMMQ2yIwhjjDENsoAwxhjToJAKCBEZLyKrRGSNiExvYHmUiLzlXv6tiPRo+ioblxf7/L8iki0iK0TkYxHp7o86G9OR9tljvQtEREUk4C+J9GafReRi95/1ShF5valrbGxe/N1OFpElIvKj++/32f6os7GIyAsiUiQiWYdYLiLymPvnsUJEhh3zh6pqSPwHhANrgZ5AJPATMKDeOtcCT7kfXwK85e+6m2CfTwVauh//MRT22b1ea+Bz4Bsg3d91N8Gfcx/gRyDe/byDv+tugn1+Bvij+/EAYIO/6z7GfT4ZGAZkHWL52cCHgADHA98e62eG0hHESGCNqq5T1X3Am8DEeutMBF52P34HOF1EpAlrbGxH3GdVXaKqe91PvwGSmrjGxubNnzPAPcBfgcqmLM5HvNnnPwCPq+ouAFUtauIaG5s3+6xAG/fjtsDmJqyv0anq58DOw6wyEfiPOr4B4kSk87F8ZigFRFcg3+N5gfu1BtdR1RpgN5DQJNX5hjf77Ol3OL+BBLIj7rP70Lubqs5rysJ8yJs/575AXxH5SkS+EZHxTVadb3izz3cCU0SkAJgP/L+mKc1vjvbf+xG1OKZyTNAQkSlAOnCKv2vxJREJAx4Bpvq5lKbWAuc001ico8TPRSRNVUv8WpVvXQq8pKoPi8ho4BURGaSqLn8XFihC6QiiEOjm8TzJ/VqD64hIC5zD0uImqc43vNlnROQM4BbgPFWtaqLafOVI+9waGAR8KiIbcM7Vzg3wgWpv/pwLgLmqWq2q64HVOIERqLzZ598BbwOo6lIgGqepXbDy6t/70QilgFgG9BGRFBGJxBmEnltvnbnAVe7HFwKfqHv0J0AdcZ9FZCjwNE44BPp5aTjCPqvqblVNVNUeqtoDZ9zlPFXN8E+5jcKbv9tzcI4eEJFEnFNO65qyyEbmzT5vAk4HEJFUnIDY3qRVNq25wJXuq5mOB3ar6pZjecOQOcWkqjUich2wEOcKiBdUdaWI3A1kqOpc4Hmcw9A1OINBl/iv4mPn5T7/HYgF/usej9+kquf5rehj5OU+BxUv93kh8CsRyQZqgZtVNWCPjr3c5xuBZ0XkzzgD1lMD+Rc+EXkDJ+QT3eMqdwARAKr6FM44y9nAGmAv8Jtj/swA/nkZY4zxoVA6xWSMMeYoWEAYY4xpkAWEMcaYBllAGGOMaZAFhDHGmAZZQBjTDIjIWBH5wN91GOPJAsIYY0yDLCCMOQoiMkVEvhOR5SLytIiEi0iZiPzDfZ+Fj0WkvXvdIe7GeCtEZLaIxLtf7y0ii0XkJxH5QUR6ud8+VkTeEZFcEXktwDsJmyBgAWGMl9ztGn4NnKiqQ3BmJF8OtMKZvTsQ+AxnhivAf4D/U9XjgEyP11/Dab09GDgB2N8OYShwA869C3oCJ/p8p4w5jJBptWFMIzgdGA4sc/9yHwMUAS7gLfc6rwKzRKQtEKeqn7lffxmnnUlroKuqzgZQ1UoA9/t9p6oF7ufLgR7Al77fLWMaZgFhjPcEeFlVZxzwosht9db7pf1rPDvp1mL/Po2f2SkmY7z3MXChiHQAEJF27nt4h+F0/wW4DPhSVXcDu0RkjPv1K4DPVLUUKBCRSe73iBKRlk26F8Z4yX5DMcZLqpotIrcCH7lvPFQN/A9QDox0LyvCGacAp3X8U+4AWEddd80rgKfdnUergYuacDeM8Zp1czXmGIlImarG+rsOYxqbnWIyxhjTIDuCMMYY0yA7gjDGGNMgCwhjjDENsoAwxhjTIAsIY4wxDbKAMMYY06D/DzGkAz+v3xxvAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plt.plot(history['loss'])\n",
        "plt.plot(history['val_loss'])\n",
        "plt.title('model loss')\n",
        "plt.ylabel('loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'test'], loc='upper left')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        },
        "id": "FPFxgxsobi4F",
        "outputId": "0b3c3df4-ae16-473b-b23f-989a000d4fcf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f691dfd0f50>"
            ]
          },
          "metadata": {},
          "execution_count": 19
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3gUV7bv/e8SEggJIUBCBIGQUABMzsFgknDOeWycbYyxseHM9Z2Zc8dz58x9z3nnvHPuGIyNAeccxzgchwGRMTkasLESEkgEkSSU43r/qLaHwQQB3Sqpe32eh0fdXdVVqxD0r2tX7b1FVTHGGBO4gtwuwBhjjLssCIwxJsBZEBhjTICzIDDGmABnQWCMMQHOgsAYYwKcBYEx9SQir4vI/1PPdXNEJPVit2NMQ7AgMMaYAGdBYIwxAc6CwPgVT5PM0yLynYiUisgrItJBRL4WkWIRSRORtietf72I7BKRQhFZLiK9Tlo2UES2eN73ARB6yr6uFZFtnveuEZF+F1jzIyKSKSLHRORzEenseV1E5FkRKRCREyKyQ0T6eJZdLSLfe2rLF5H/cUF/YcZgQWD80y3AJCAFuA74GvhXoD3Ov/knAUQkBXgPmOFZ9hXwhYg0F5HmwKfAW0A74CPPdvG8dyDwKvAoEAXMBz4XkRbnU6iITAD+X+B2oBOQC7zvWXw5cJnnOCI96xz1LHsFeFRVI4A+wNLz2a8xJ7MgMP5ojqoeUtV8YBWwXlW3qmoFsBAY6FnvDuBLVV2sqtXAfwEtgVHACCAEmKWq1ar6MbDxpH1MAear6npVrVXVN4BKz/vOx93Aq6q6RVUrgd8BI0UkHqgGIoCegKjqD6p6wPO+auASEWmtqsdVdct57teYn1kQGH906KTH5ad53srzuDPON3AAVLUO2AfEepbl6z+Pyph70uNuwK89zUKFIlIIdPW873ycWkMJzrf+WFVdCjwPvAAUiMgCEWntWfUW4GogV0RWiMjI89yvMT+zIDCBbD/OBzrgtMnjfJjnAweAWM9rP4k76fE+4N9Vtc1Jf8JU9b2LrCEcp6kpH0BVn1PVwcAlOE1ET3te36iqNwAxOE1YH57nfo35mQWBCWQfAteIyEQRCQF+jdO8swZYC9QAT4pIiIjcDAw76b0vAVNFZLjnom64iFwjIhHnWcN7wAMiMsBzfeE/cJqyckRkqGf7IUApUAHUea5h3C0ikZ4mrRNA3UX8PZgAZ0FgApaq/ghMBuYAR3AuLF+nqlWqWgXcDNwPHMO5nvDJSe/dBDyC03RzHMj0rHu+NaQBzwB/wzkLSQTu9CxujRM4x3Gaj44Cf/EsuwfIEZETwFScaw3GXBCxiWmMMSaw2RmBMcYEOAsCY4wJcBYExhgT4CwIjDEmwAW7XcD5io6O1vj4eLfLMMaYJmXz5s1HVLX96ZY1uSCIj49n06ZNbpdhjDFNiojknmmZNQ0ZY0yAsyAwxpgAZ0FgjDEBrsldIzid6upq8vLyqKiocLsUnwsNDaVLly6EhIS4XYoxxk/4RRDk5eURERFBfHw8/zxYpH9RVY4ePUpeXh4JCQlul2OM8RN+0TRUUVFBVFSUX4cAgIgQFRUVEGc+xpiG4xdBAPh9CPwkUI7TGNNw/CYIjDHGb1UUwbfPQe4an2zegsALCgsLmTt37nm/7+qrr6awsNAHFRlj/ELhXvjmX+GvvWHxM5CxyCe78YuLxW77KQimTZv2T6/X1NQQHHzmv+KvvvrK16UZY5qi/M2w5nn4/jMQgd43w8jHofMAn+zOgsALfvvb35KVlcWAAQMICQkhNDSUtm3bsnv3btLT07nxxhvZt28fFRUVPPXUU0yZMgX4x3AZJSUlXHXVVYwePZo1a9YQGxvLZ599RsuWLV0+MmNMg6mrg/SvnQDYuwZatHY+/Ic/CpFdfLprvwuCf/tiF9/vP+HVbV7SuTX/+7reZ1z+5z//mZ07d7Jt2zaWL1/ONddcw86dO3++xfPVV1+lXbt2lJeXM3ToUG655RaioqL+aRsZGRm89957vPTSS9x+++387W9/Y/LkyV49DmNMI1RVBtvfhbVz4VgWRMbBFf8BA++B0NYNUoLfBUFjMGzYsH+6z/+5555j4cKFAOzbt4+MjIxfBEFCQgIDBjinfYMHDyYnJ6fB6jXGuKCkADa8BBtfhvJj0HkQ3Poq9LoBmjXsR7PfBcHZvrk3lPDw8J8fL1++nLS0NNauXUtYWBjjxo07bT+AFi1a/Py4WbNmlJeXN0itxpgGVrAb1j4P330ItVXQ42oY9QTEjXSuB7jA74LADRERERQXF592WVFREW3btiUsLIzdu3ezbt26Bq7OGOM6Vdizwmn/z1wMwS1h4GQYMQ2ik9yuzoLAG6Kiorj00kvp06cPLVu2pEOHDj8vu/LKK5k3bx69evWiR48ejBgxwsVKjTENqqYKdn3iBMChHRAeA+N/D0MehPCoc7+/gYiqul3DeRkyZIieOjHNDz/8QK9evVyqqOEF2vEa0+SUH4fNr8P6+VB8ANr3hJFPQN/bICTUlZJEZLOqDjndMjsjMMYYbzmeA+tehC1vQXUpJIyF65+HpImutf/XhwWBMcZcrH0bYe0c+OELkCDoc6vTB6BTP7crqxcLAmOMuRB1tbD7S+cOoH3rITQSRj3pdABr3dnt6s6LBYExxpyPqlLY+g6smwvH90CbbnDlfzp3AbVo5XZ1F8SCwBhj6qP4IGxYABtfgYpC6DIUUv8Iva6DoGZuV3dRfBoEIpIDFAO1QM2pV6xFpC3wKpAIVAAPqupOX9ZkjDHn5dAuWPuC0wGsrgZ6XQsjp0PccLcr85qGGIZ6vKoOOMNtS/8KbFPVfsC9wOwGqMfrLnQYaoBZs2ZRVlbm5YqMMRdFFTKXwFs3wYujYNdCGPIAPLkF7njbr0IA3J+P4BJgKYCq7gbiRaTD2d/S+FgQGOMnaiqd9v8XL4W3b3bOBib+AWbugqv/Au26u12hT/j6GoECi0REgfmquuCU5duBm4FVIjIM6AZ0AQ6dvJKITAGmAMTFxfm45PN38jDUkyZNIiYmhg8//JDKykpuuukm/u3f/o3S0lJuv/128vLyqK2t5ZlnnuHQoUPs37+f8ePHEx0dzbJly9w+FGMCU9kx2PSqcw2g5BDE9IYb5kLfWyG4xbnf38T5OghGq2q+iMQAi0Vkt6quPGn5n4HZIrIN2AFsxbme8E88AbIAnJ7FZ93j17+Fgzu8Vb+jY1+46s9nXHzyMNSLFi3i448/ZsOGDagq119/PStXruTw4cN07tyZL7/8EnDGIIqMjOSvf/0ry5YtIzo62rs1G2PO7Vi2M/zztnegugwSJ8CNLzo/G3EHMG/zaRCoar7nZ4GILASGAStPWn4CeABAnFnZ9wDZvqzJ1xYtWsSiRYsYOHAgACUlJWRkZDBmzBh+/etf85vf/IZrr72WMWPGuFypMQFs73pY85zTDyAoGPrd7nQA6+D+6MVu8FkQiEg4EKSqxZ7HlwN/OmWdNkCZqlYBDwMrPeFw4c7yzb0hqCq/+93vePTRR3+xbMuWLXz11Vf8/ve/Z+LEifzhD39woUJjAlRtDez+wrkDKG8jhLaBMf8Cw6ZAREe3q3OVL88IOgALnS/6BAPvquo3IjIVQFXnAb2ANzzXEHYBD/mwHp85eRjqK664gmeeeYa7776bVq1akZ+fT0hICDU1NbRr147JkyfTpk0bXn755X96rzUNGeMjlcWw9W1nDKDCXGibAFf/Fwy4C5qHn/v9AcBnQaCq2UD/07w+76THa4EUX9XQUE4ehvqqq67irrvuYuTIkQC0atWKt99+m8zMTJ5++mmCgoIICQnhxRdfBGDKlClceeWVdO7c2S4WG+NNJ/Y7o39ueg0qi6DrCLji352JYJp4BzBvs2Gom6BAO15jzsvBHc74/zs/Bq2DXtfDqOnQ5bQjMAcMG4baGOPfVCEzDdbMcWYCCwmHoQ/DiMegbbzb1TV6FgTGmKarugJ2fOhcAD68GyI6OeP/DL4fWrZ1ubimw2+CQFWRALjvt6k15RnjE6VHYdMrTgew0sPQoS/cNB963wzBzd2ursnxiyAIDQ3l6NGjREVF+XUYqCpHjx4lNNSdqe6Mcd2RTFj3Amx7D2rKIWkSjHrCmQnMj//v+5pfBEGXLl3Iy8vj8OHDbpfic6GhoXTp0sXtMoxpOKqQu8aZAObHr6FZCPS7w5kDOKan29X5Bb8IgpCQEBISEtwuwxjjTbU18MNnzh1A+7dAy3Zw2dMw7BFoFeN2dX7FL4LAGONHKk7A1rdg3Two2gvtEuGav0L/X0HzMLer80sWBMaYxqEoD9bPg81vQOUJiBvlDBmTchUEuT1ivn+zIDDGuGv/Nqf9f9dC53rAJTc4F4BjB7tdWcCwIDDGNLy6OshY5ARAzipoHgHDHoURU6FN45tzxN9ZEBhjGk51OWx/H9bNhSPp0DoWJv0fGHwfhEa6XV3AsiAwxvheyWHY+LLzp+wIdOoPN78MvW90bgc1rrIgMMb4zuH0f3QAq62ElCud+//jR1sHsEbEgsAY412qkLPaaf9P/waatYABv4IRj0P7Jj/qvF+yIDDGeEdtNez6FNbOgQPbISwKxv7WGQW0VXu3qzNnYUFgjLk4FUXOvf/r58GJfIhOgetmO8NAhLR0uzpTDxYExpgLU7jX6f275U2oKob4MXDts85AcNYBrEmxIDDGnJ/8zc74P99/5jzvc7NzAbjzAHfrMhfMgsAYc251dZD+tRMAe9dAi9YwchoMnwqRNhpuUxcwQVBVU8eWvccZ0T3K7VKMaTqqymD7u7B2LhzLgsiucMV/wMB7ILS129UZLwmYIPh0az7/82/fMTyhHTMnpVggGHM2JQWw4SWnA1j5Meg8CG59FXrdAM0C5mMjYATMb/T6AZ0praph7vIs7lywjpHdo5g5KYVhCe3cLs2YxqNgt3P//3cfQm0V9LgaRj4O3UZZBzA/Jk1tDtwhQ4bopk2bLvj9FdW1vLN+Ly8uz+JISSWXJkUxMzWFIfEWCCZAqcKeFU77f+ZiCG4JA+6CEdMgOsnt6oyXiMhmVR1y2mWBFgQ/Ka+q5Z31ucxbkcWRkirGJEczIzWFwd3aeqFKY5qAmirY9YlzBnBwB4S3h2FTYMhDEG5Np/7GguAsyqpqeHtdLvNXZHO0tIrLUtozMzWZgXEWCMZPlR+Hza/D+vlQfADa93Saf/reDiGhbldnfMSCoB7Kqmp4c20uC1Zmc6y0inE92jMjNYUBXdt4fV/GuOJ4Dqx7Eba8BdWlkDAWRk2HpFRr/w8ArgWBiOQAxUAtUHNqESISCbwNxOFcuP4vVX3tbNv0VRD8pLSyhjfW5rBgZTaFZdVM6BnDjNRk+nWxQDBN1L6Nzvg/P3wBEgR9bnXOADr1c7sy04DcDoIhqnrkDMv/FYhU1d+ISHvgR6CjqladaZu+DoKflFTW8MYaJxCKyqtJ7RXDjNQU+sTa5BmmCairhR+/gjVzYN96aBEJQx6A4Y9C685uV2dccLYgcPv2UQUiRESAVsAxoMbdkhytWgTz+Pgk7h3Zjde/zeGlVdlcO2c1ky7pwIzUZHp3tkAwjVBVKWx7F9a+AMf3QJtucOV/wsDJ0KKV29WZRsrXZwR7gOM4H/jzVXXBKcsjgM+BnkAEcIeqfnma7UwBpgDExcUNzs3N9VnNZ3KioprXVufw8upsiitquKJ3B2akptCrk/WuNI1A8UHYsAA2vgIVhdBlqDP+T89rrQOYAdxtGopV1XwRiQEWA9NVdeVJy28FLgX+BUj0rNNfVU+caZsN1TR0JkXl1by6eg+vrt5DcWUNV/XpyFOpyfTsaIFgXHBol/Pt/7sPoa4Gel4Do56EuOFuV2YaGdeahlQ13/OzQEQWAsOAlSet8gDwZ3XSKNNzBtET2ODLui5GZMsQZk5K4cFLE3hldTavfpvD1zsPck3fTjyVmkxKhwi3SzT+ThWyljr3/2cthZAwGHw/jHgMohLdrs40QT4LAhEJB4JUtdjz+HLgT6estheYCKwSkQ5ADyDbVzV5U2RYCP9yeQ8eHJ3Ay6v28Nq3e/hq5wEnECYmk2yBYLytphJ2fOycARTsglYdYMIzMORBCLOe8ebC+axpSES6Aws9T4OBd1X130VkKoCqzhORzsDrQCdAcM4O3j7bdt1uGjqT46VVvLQqmzfW5FBWXct1/Trz5MRkkmLsAp25SGXHYNOrziBwJQch5hKn/b/vrRDcwu3qTBNhHcoa0LHSKhaszObNtTlUVNdyfX8nELq3t0Aw5+lYtjP887Z3oLoMEic4AZA4wTqAmfNmQeCCoyWVnkDIpbKmlhsHxDJ9YjIJ0eFul2Yau73rYc1zsPtLCAqGfrc7HcA69Ha7MtOEWRC46EhJJfNXZPHWulyqa5UbB8Ty5MQkukVZIJiT1NbA7i+c9v+8jRDaBoY+5AwCF9HR7eqMH7AgaAQKiiuYvyKbt9flUlOn3DwwlukTkomLCnO7NOOmyhLY+jasmwuFudA23mn+GXAXNLcvC8Z7LAgakYITFby4Iot31u+lrk65ZVAXnpiQRNd2FggB5cR+Z/TPza9BRRF0He7pAHYNBDVzuzrjhywIGqFDJyp4cXkW725wAuG2IV14fHwSXdpaIPi1gzucCWB2fgxaB72ug5HToetQtyszfs6CoBE7WFTB3OWZvL9hH4py25CuPD4+idg2Ld0uzXiLKmSmOQPA7VkBIeEw6B4YPhXaJbhdnQkQFgRNwP7CcuYuz+SDjfsAuGOoEwidIi0QmqzqCtjxoXMB+PBuiOjkjP45+H5oaRMfmYZlQdCE5BeW88KyTD7atA9BuHNYV6aNS6JjpM0c1WSUHoVNrziDwJUehg59YdQT0PtmCG7udnUmQFkQNEF5x8s8gZBHUJBw17A4HhuXSIfWFgiN1pFMWPcCbHsPasohaZITAAljrQOYcZ0FQRO271gZzy/N5OMteQQHCXcNj+OxsYnEWCA0Dqqwd61zAfjHr6BZCPS7w+kAFtPL7eqM+ZkFgR/Ye7SMOUsz+GRrPsFBwuQR3Zg6NpH2ETbWjCtqa+CHz5wA2L8FWrZzOoANfQQiOrhdnTG/YEHgR3KOlDJnaSYLt+bRPDiIe0Z049GxiUS3skBoEBUnYOtbsG4eFO2Fdokwchr0vwua262/pvGyIPBDe46UMmdJBp9uy6dFcDPuHdmNKZd1J8oCwTeK8mD9PNj8BlSegLhRTvt/ylUQFOR2dcackwWBH8s6XMKcJRl8vn0/oSHNuHdkPFMu6067cLs7xSv2b3MmgNm10LkecMkNTgDEDna7MmPOiwVBAMgsKOG5JRl88d1+wkKacd+oeB4Z0522Fgjnr64OMhY5AZCzCpq3gkH3OX0A2nZzuzpjLogFQQDJOFTM7CUZfLnjAOHNg7l/VDwPj0mgTZgFwjlVl8P2950B4I6kQ+tYp/fv4PsgNNLt6oy5KBYEASj9UDGz05xAaNUimAcujefh0d2JDAtxu7TGp+QwbHzZ+VN2BDr2cyaA732jczuoMX7AgiCA7T54gtlpGXy98yARLYJ5cHQCD45OILKlfcBxOP0fHcBqKyH5Cqf9P36MdQAzfseCwPD9/hPMXpLO33cdIiI0mIdHd+eB0fG0Dg2wQFCFnNVO+3/6N9CsBfS/0+kA1r6H29UZ4zMWBOZnu/YXMSstg8XfH6J1aDCPjOnO/ZfGE+HvgVBbDbs+hbVz4MB2CItyOn8NfRhatXe7OmN8zoLA/MLO/CJmpaWT9kMBbcJCeGRMd+4bFU+rFsFul+ZdFUXOvf/r58OJPIhKdr79978TQmxkVxM4LAjMGX2XV8istAyW7i6gbVgIj1zWnftGxhPe1AOhcK/T+3fLm1BV7LT7j3wCki+3DmAmIFkQmHPatq+Q2WnpLPvxMO3CmzPlsu7cO7IbYc2bWCDkb3bG//n+M+d5n5udM4DOA92tyxiXWRCYetu69ziz0jJYkX6YqPDmPDq2O5NHNPJAqKuD9K+dANi7Blq0du79Hz4VIru4XZ0xjYIFgTlvm3OPMystnVUZR4hu1ZypYxO5e3g3WjZvRBOrV5XB9vecGcCOZUFkV+fDf9C9ENra7eqMaVQsCMwF25RzjFlpGazOPEJ0qxY8Ni6Ru4fHERriYiCUFMCGl5wOYOXHnGafkU/AJTdCs0Z85mKMiywIzEXbsOcYs9LSWZN1lJgIJxB+NayBA6Fgt3P//3cfQm0V9LjKCYBuo6wDmDHn4FoQiEgOUAzUAjWnFiEiTwN3e54GA72A9qp67EzbtCBw17rso8xKS2dd9jE6tG7BtHFJ3DG0q+8CQRX2rHDa/zMXQ3AoDLgLRjwO0Um+2acxfsjtIBiiqkfqse51wExVnXC29SwIGoc1WUeYtTiDDTnH6Ng6lMfHJ3L70K60CPZSINRUwa5PnDOAgzsgvD0MmwJDHoLwKO/sw5gAcrYgaEwNqr8C3nO7CFM/oxKjGdk9ijVZR3l2cTrPfLaLF5dnMW18ErcP6Urz4Au8V7+8EDa/5nQAKz4A7XvC9XOg7+0QYvM0G+MLvj4j2AMcBxSYr6oLzrBeGJAHJJ2tWQjsjKAxUlVWZx7h2cXpbNlbSGybljw+PolbB3epfyAcz4F1L8KWt6C6FBLGwqjpkDjROoAZ4wVuNg3Fqmq+iMQAi4HpqrryNOvdAUxW1evOsJ0pwBSAuLi4wbm5uT6r2Vw4VWVlhhMI2/Y5gTB9QhK3DO5CSLMzfJjv2+iM//PDFyBB0OcW5wJwp34NW7wxfq5R3DUkIn8ESlT1v06zbCHwkaq+e67t2BlB46eqLE8/zKzF6WzPK6Jru5ZMH5/MTYNinUCoq4Ufv4I1c2DfemgRCUPuh2GPQmSs2+Ub45dcCQIRCQeCVLXY83gx8CdV/eaU9SKBPUBXVS0913YtCJoOVWXZjwXMSsvgu7wiUtoG8efEHQzMfxc5vgfaxMGIaTBwMrSIcLtcY/zaRV8sFpGngNdwbgV9GRgI/FZVF53lbR2AheLc3x0MvKuq34jIVABVnedZ7yZgUX1CwDQtIsKEnh0Y37mOnK9nEb37bSJ2lvB9UAonBv+VIVfeQ3CITaFpjNvqdUYgIttVtb+IXAE8CjwDvKWqg3xd4KnsjKAJObTLGf5hx0dQW432vIaNne7ij1sj+P5gMQnR4Tw5MYnr+8fSLMg6hBnjS964ffSn/6VX4wTALhHrymlOQxWyljr3/2cthZAwGHQfjHgMiUpkGPDlZcrfdx1iVlo6Mz/YzpylmTw1MZlr+3W2QDDGBfU9I3gNiAUSgP5AM2C5qg72bXm/ZGcEjVRNJez42DkDKNgFrTp4OoA9CGHtTvuWujrl77sOMistgx8PFZMU04onJyZzTd9OFgjGeNlFXywWkSBgAJCtqoUi0g7ooqrfebfUc7MgaGTKjsGmV51B4EoOQswlzvj/fW+D4Bb12kRdnfL1zoPMXpJO+qESkmNa8VRqMlf36USQBYIxXuGNILgU2KaqpSIyGRgEzFbVBr+h34KgkTiWDWvnwrZ3oLoMuo+HUU84HcAusNWwrk75cscBZi/JILOghB4dIngqNZkre3e0QDDmInkjCL7DaRLqB7yOc+fQ7ao61ot11osFgcv2rvd0APtvCAp2vvmPfBw69vHaLmrrlP/+bj/PLckg63ApPTtGMCM1mcsvsUAw5kJ5Iwi2qOogEfkDkK+qr/z0mreLPRcLAhfU1To9f9c+D3kbIbSN0/Y/bAq07uSz3dbWKV9sdwIh+0gpvTq19gRCB+xeBWPOjzfuGioWkd8B9wBjPNcMQrxVoGmkKktg69uwbi4U5kLbeLjqL84w0C1a+Xz3zYKEGwfGcm2/TnzuCYRH39pM786tmZGaQmqvGAsEY7ygvmcEHYG7gI2qukpE4oBxqvqmrws8lZ0RNIAT+53RPze/BhVF0HW4M/5Pz2sgyL2ZyWpq6/h0237mLM0g92gZfWMjmZGazISeFgjGnItXhpgQkQ7AUM/TDapa4KX6zosFgQ8d3OFMALPzY9A66HUdjJwOXYee+70NqKa2jk+25jNnaQb7jpXTv0skM1JTGNejvQWCMWfgjWsEtwN/AZbjdC4bAzytqh97sc56sSDwMlXITHMGgNuzAkLCnbF/RjwG7RLcru6sqmvr+GRLHnOWZpJ3vJwBXdswIzWZsSkWCMacyhtBsB2Y9NNZgIi0B9JUtb9XK60HCwIvqa6AHR86HcAO74aITp4OYA9Ay7ZuV3deqmrq+NuWPJ5fmkl+YTkD49owMzWFMcnRFgjGeHjjYnHQKU1BRwGbLaQpKj3q6QC2AEoLoEMfuHGeMw9AcNMcAK55cBC/GhbHLYO68PHmPF5Ylsm9r25gcLe2zExN4dKkKAsEY86ivkHwjYj8nX9MJXkH8JVvSjI+cSQT1r0A296DmnJISnUuAHcfd8EdwBqb5sFB3DU8jlsGx/LRJicQJr+ynqHxTiCMTLRAMOZ0zudi8S3ApZ6nq1R1oc+qOgtrGjoPqrB3rXMB+MevoFkI9LvdCYCYXm5X53OVNbV8uHEfLyzL4uCJCoYltPs5EIwJNI1ihjJvsSCoh9oa+OEzJwD2b3Ha/Ic+DEMfgYgOblfX4Cqqa/lg4z7mLs/k0IlKRnR3AmF4dwsEEzguOAhEpBhn4vlfLAJUVVt7p8T6syA4i8pi2PImrJsHRXuhXSKMnAb974LmYW5X57qK6lre27CXucuzOFxcyajEKGZOSmFo/OlHRzXGn9gZgb8ryoP182DzG1B5AuJGOQPApVzpagewxqqiupZ31u/lxeVZHCmpZHRSNDMnJTO4mwWC8V8WBP5q/zZn/J9dC50OYJfc4HQA69Lg00Q0SeVVtbyzPpd5K7I4UlLFmORoZk5KYVBc07p91pj6sCDwJ3V1kLHICYCcVdC8FQy6F4ZPhbbd3K6uSSqrquHtdbnMX5HN0dIqxqa0Z+akFAZ0beN2acZ4jQWBP6guh+3vOwPAHUmHiM4wYqozDWRL+8DyhtLKGt5al8v8FVkcL6tmfI/2zEhNob8FgvEDFgRNWekR2PiyMwNY2RHo2C2Xhg4AABUHSURBVA9GTYfeNzm3gxqvK62s4Y21OSxYmU1hWTUTe8YwIzWFvl0i3S7NmAtmQdAUHU53OoBtfx9qKiD5CucCcPwYv+kA1tiVVNbwxhonEIrKq0nt1YEZqcn0ibVAME2PBUFToQo5q532//RvoFkL6H+nMwNY+x5uVxewiiuqef3bHF5alc2Jihouv6QDM1JTuKRzg989bcwFsyBo7GqrYdenzhSQB7ZDWJTT+Wvow9CqvdvVGY8TFdW8tjqHl1dnU1xRw5W9O/JUajK9OlkgmMbPgqCxqihy7v1fPx9O5EFUkjP8Q/87IaSl29WZMygqr+aV1Xt4bfUeiitruLpvR56amEKPjhFul2bMGVkQNDaFe53ev1vehKpi6Dbaaf9PvgKCbFDXpqKorJqXV2fz2rc5lFbVcHXfTsyYmExyBwsE0/hYEDQW+Zud8X++/8x53vsmJwA6D3S3LnNRCsuqeGlVNq9/m0NZdS3X9uvMUxOTSIqxQDCNhwWBm+rqIP1rJwD2roHmETD4PqcDWJuubldnvOhYqRMIb6zJoby6luv7d+bJickktm/ldmnGuBcEIpIDFAO1QM3pihCRccAsIAQ4oqpjz7bNJhMEVWWw/T1nBrBjWRDZ1fnwH3QvhNrFRX92tKSSBauyeXNNLpU1tdwwIJbpE5LoboFgXOR2EAxR1SNnWN4GWANcqap7RSTmlJnQfqHRB0FJgdP5a+PLUH7MafYZ+YQzDpB1AAsoR0oqWbAymzfX5lBVU8eNA2N5ckIy8dHhbpdmAlBjDoJpQGdV/X19t9log6Bgt3P//3cfQm0V9LjKCYBuo6wDWIA7XFzJ/BVZvL0+l+pa5aaBzhlCtygLBNNw3AyCPcBxnDkN5qvqglOW/9Qk1BuIAGar6pun2c4UYApAXFzc4NzcXJ/VfF5UYc8Kp/0/czEEh0L/XzkdwKKT3a7ONDIFxRXMW57NO+tzqalTbhkUy/QJyXRtZ3NFGN9zMwhiVTVfRGKAxcB0VV150vLngSHARKAlsBa4RlXTz7TNRnFGUFMFuz5xzgAO7oDw9p4OYA9BeLS7tZlGr+BEBXOXZ/Huhr3U1Sm3Du7C4+OTLBCMT50tCOo7ef0FUdV8z88CEVkIDANWnrRKHnBUVUuBUhFZCfQHzhgEriovhM2vOR3Aig9AdA+47jnodweEhLpdnWkiYlqH8sfrezN1bCIvLs/kvQ37+HhzHrcN6coTE5KIbWOdCU3D8tkZgYiEA0GqWux5vBj4k6p+c9I6vYDngSuA5sAG4E5V3Xmm7bpyRnA8B9a9CFvegupSSLjMmQAmKdU6gJmLdqConLnLsvhg4z4U5fYhXXl8fBKdLRCMF7nSNCQi3YGFnqfBwLuq+u8iMhVAVed51nsaeACoA15W1Vln226DBkHeJljzHPzwBUgQ9LnFaf/v1L9h9m8Cyv7Ccl5YlsmHm/YhCHcM7cq08Yl0irRAMBfPOpSdj7pa+PEr5wLwvnXQIhKG3A/DHoXIWN/t1xiPvONlzF2exUeeQPjVsK5MG59Eh9bW/GgunAVBfVSVwrZ3nQ5gx/dAmzgYMQ0GToYWNlSAaXh5x8t4YVkmH23KIyhIuGtYHNPGJRJjgWAugAXB2RQfhA0LYNOrUH4cYgc7M4D1vA6a+fRaujH1su9YGc8vzeTjLXkEBwl3D+/G1HHdiYmwQDD1Z0FwOod2Od/+d3zkzAfQ8xqnA1jcCOsAZhqlvUfLmLM0g0+25hPSTJg8vBuPjk2kfUQLt0szTYAFwU9UIWupc/9/1lIIbgkD73aagKISvVuoMT6Sc6SUOUszWbg1j+bBQdw7Mp4pl3UnupUFgjkzCwJwpoD86n9CwS4Ij4HhU2DIQxDWzvtFGtMA9hwpZc6SDD7dlk+L4GbcO6obj16WSLvw5m6XZhohCwKA/C3w6TRn/P++t0GwfXsy/iHrcAlzlmTw2fb9tAxpxn2j4pkypjttLRDMSSwIfqJq7f/Gb2UWlPDckgy++G4/YSHNuP/SeB4Z0502YRYIxoLAmICScaiY2Usy+HLHAcKbB/PApfE8PLo7kWE2DHogsyAwJgD9eLCY5zyBENEimAdGJ/DQ6AQiW1ogBCILAmMC2O6DJ5idlsHXOw8SERrMQ6MTeHB0Aq1DLRACiQWBMYbv959g9pJ0/r7rEK1Dg3l4THceuDSeCAuEgGBBYIz52c78ImYvyWDx94eIbBnCI2MSuP/SBFq1sJ70/syCwBjzCzvzi5iVlk7aDwW0CQvhkTHduW9UvAWCn7IgMMac0Xd5hcxKy2Dp7gLahoUw5bJE7h3ZjXALBL9iQWCMOadt+wqZlZbO8h8P0y68OY9e1p17RnYjrLkFgj+wIDDG1NuWvceZlZbByvTDRLdqzqOXJTJ5RDdaNm/mdmnmIlgQGGPO2+bcY8xKy2BVxhGiW7Vg6tjuTB7RjdAQC4SmyILAGHPBNuYcY1ZaOt9mHqV9RAseG5vIXcPjLBCaGAsCY8xFW599lFlpGazNPkpMRAumjUvkzmEWCE2FBYExxmvWZh3l2bR0Nuw5RsfWoUwbn8gdQ7vSItgCoTGzIDDGeJWq/hwIG3OO0ykylGnjk7h9SBcLhEbKgsAY4xOqyreZTiBszj1O58hQHp+QxG2Du9I8OMjt8sxJLAiMMT6lqqzKOMKzaels3VtIbJuWPDEhiVsHdyGkmQVCY2BBYIxpEKrKivTDPJuWwfZ9hXRp25LpE5K4eZAFgtssCIwxDUpVWZ5+mFmL09meV0RcuzCemJDEzQNjCbZAcIUFgTHGFarKsh8LeHZxBjvyi+gWFcb0CcncOKCzBUIDsyAwxrhKVVnyQwHPpqWza/8JEqLDmT4hiev7WyA0FNeCQERygGKgFqg5tQgRGQd8BuzxvPSJqv7pbNu0IDCm6VJVFn9/iFlpGXx/4ATdo8N5cmIy1/XvTLMgcbs8v+Z2EAxR1SNnWD4O+B+qem19t2lBYEzTV1enLPr+ELPS0tl9sJjE9k4gXNvPAsFXzhYEdk5mjGlwQUHClX068tWTY3jx7kEEBwXx1PvbuGLWSr7Yvp+6uqbVZN3U+ToIFFgkIptFZMoZ1hkpIttF5GsR6X26FURkiohsEpFNhw8f9l21xpgGFRQkXNW3E18/NYYX7hpEkMD097Zy5eyVfPndAQuEBuLrpqFYVc0XkRhgMTBdVVeetLw1UKeqJSJyNTBbVZPPtk1rGjLGf9XVKV/uOMDsJRlkFpTQs2MET01M5oreHQmyJqOL4lrTkKrme34WAAuBYacsP6GqJZ7HXwEhIhLty5qMMY1XUJBwXf/O/H3GZcy+cwBVtXU89s4Wrn5uFd/sPEhTu8uxqfBZEIhIuIhE/PQYuBzYeco6HUVEPI+Heeo56quajDFNQ7Mg4YYBsSyeOZZZdwygqqaOqW9v5prnVrNolwWCt/lyMtIOwELP53ww8K6qfiMiUwFUdR5wK/CYiNQA5cCdar9hY4xHsyDhxoGxXNuvE59v389zSzKY8tZm+sS2ZsbEFCb2isHzGWMugnUoM8Y0GTW1dXy6bT9zlmaQe7SMfl0imZGazPgeFgjnYj2LjTF+pbq2joVb85mzNIN9x8rp37UNM1KTGZfS3gLhDCwIjDF+qbq2jk+25DFnaSZ5x8sZ0LUNMyelcFlytAXCKSwIjDF+raqmjr9tyeP5pZnkF5YzKM4JhNFJFgg/sSAwxgSEqpo6Ptq8jxeWZrK/qIIh3doyc1IKoxKjAj4QLAiMMQGlsqaWDzflMXdZJgeKKhgW344Zk5IZlRi43ZQsCIwxAamyppYPNu7jhWWZHDpRyfCEdsyclMKI7lFul9bgLAiMMQGtorqW9zfsZe7yLAqKKxnZPYqZk1IYltDO7dIajAWBMcbgBMK76/fy4oosDhdXcmlSFDNTUxgS7/+BYEFgjDEnKa+q5Z31ucxbkcWRkirGJEczIzWFwd3aul2az1gQGGPMaZRX1fL2OicQjpZWcVlKe2amJjMwzv8CwYLAGGPOoqyqhrfW5jJ/ZTbHSqsY16M9M1NT6N+1jduleY0FgTHG1ENpZQ1vrs1lwcosjpdVM6FnDDNTU+jbJdLt0i6aBYExxpyHksoa3liTw0ursiksqya1VwwzUlPoE9t0A8GCwBhjLkBxRbUnEPZQVF7NpEs6MCM1md6dm14gWBAYY8xFOFFRzevf5vDyqmxOVNRwRe8OzEhNoVen1m6XVm8WBMYY4wVF5dW89u0eXlm9h+KKGq7q05GnUpPp2bHxB4IFgTHGeFFReTWvrN7Da6v3UFxZwzV9O/FUajIpHSLcLu2MLAiMMcYHCsuqnED4NofSKk8gTEwmuREGggWBMcb40PHSKl5enc3r3+ZQVl3Ldf068+TEZJJiWrld2s8sCIwxpgEcK63ipVXZvLEmh4rqWq7v7wRC9/buB4IFgTHGNKCjJZUsWJXNm2tyqayp5cYBsUyfmExCdLhrNVkQGGOMC46UVLJgZTZvrs2hula5cUAsT05MoltUwweCBYExxrjocHEl81dk8da6XGrqlJsHxjJ9QjJxUWENVoMFgTHGNAIFxRXMW57NO+tzqa1TbhnUhScmJNG1ne8DwYLAGGMakYITFcxdnsW7G/ZSV6fcNqQLj49Poktb3wWCBYExxjRCB4sqeHF5Ju9t2Iei3DakK4+PTyK2TUuv78uCwBhjGrEDReXMXZbFBxudQLhjqBMInSK9FwgWBMYY0wTsLyznhWWZfLhpH4Jw57CuTBuXRMfI0IvetmtBICI5QDFQC9ScsQiRocBa4E5V/fhs27QgMMb4u7zjZbywLIuPNu0jKEi4a1gcj41LpEPrCw8Et4NgiKoeOcs6zYDFQAXwqgWBMcY49h0r44VlmXy8OY9mQcLTV/Tg4THdL2hbZwuCoIuq0jumA38DCtwuxBhjGpOu7cL48y39WPrrcdwwoLPP7ioK9slW/0GBRSKiwHxVXXDyQhGJBW4CxgNDz7QREZkCTAGIi4vzXbXGGNMIxUWF8f/d2t9n2/f1GcFoVR0EXAU8LiKXnbJ8FvAbVa0720ZUdYGqDlHVIe3bt/dVrcYYE5B8ekagqvmenwUishAYBqw8aZUhwPsiAhANXC0iNar6qS/rMsYY8w8+CwIRCQeCVLXY8/hy4E8nr6OqCSet/zrw3xYCxhjTsHx5RtABWOj5th8MvKuq34jIVABVnefDfRtjjKknnwWBqmYDv7i6caYAUNX7fVWLMcaYM2sMt48aY4xxkQWBMcYEOAsCY4wJcE1u0DkROQzkXuDbo4EzDnfhp+yYA4Mdc2C4mGPupqqn7YjV5ILgYojIpjONteGv7JgDgx1zYPDVMVvTkDHGBDgLAmOMCXCBFgQLzr2K37FjDgx2zIHBJ8ccUNcIjDHG/FKgnREYY4w5hQWBMcYEOL8MAhG5UkR+FJFMEfntaZa3EJEPPMvXi0h8w1fpXfU45n8Rke9F5DsRWSIi3dyo05vOdcwnrXeLiKiINPlbDetzzCJyu+d3vUtE3m3oGr2tHv+240RkmYhs9fz7vtqNOr1FRF4VkQIR2XmG5SIiz3n+Pr4TkUEXvVNV9as/QDMgC+gONAe2A5ecss40YJ7n8Z3AB27X3QDHPB4I8zx+LBCO2bNeBM4cGOtw5s92vXYf/56Tga1AW8/zGLfrboBjXgA85nl8CZDjdt0XecyXAYOAnWdYfjXwNSDACGD9xe7TH88IhgGZqpqtqlXA+8ANp6xzA/CG5/HHwETxjJfdRJ3zmFV1maqWeZ6uA7o0cI3eVp/fM8D/Af4TqGjI4nykPsf8CPCCqh4HZ1KoBq7R2+pzzAq09jyOBPY3YH1ep6orgWNnWeUG4E11rAPaiEini9mnPwZBLLDvpOd5ntdOu46q1gBFQFSDVOcb9Tnmkz2E842iKTvnMXtOmbuq6pcNWZgP1ef3nAKkiMi3IrJORK5ssOp8oz7H/EdgsojkAV8B0xumNNec7//3c/L15PWmkRGRyThThI51uxZfEpEg4K/A/S6X0tCCcZqHxuGc9a0Ukb6qWuhqVb71K+B1Vf2/IjISeEtE+ug55kI3/+CPZwT5QNeTnnfxvHbadUQkGOd08miDVOcb9TlmRCQV+F/A9apa2UC1+cq5jjkC6AMsF5EcnLbUz5v4BeP6/J7zgM9VtVpV9wDpOMHQVNXnmB8CPgRQ1bVAKM7gbP6qXv/fz4c/BsFGIFlEEkSkOc7F4M9PWedz4D7P41uBpeq5CtNEnfOYRWQgMB8nBJp6uzGc45hVtUhVo1U1XlXjca6LXK+qm9wp1yvq82/7U5yzAUQkGqepKLshi/Sy+hzzXmAigIj0wgmCww1aZcP6HLjXc/fQCKBIVQ9czAb9rmlIVWtE5Ang7zh3HLyqqrtE5E/AJlX9HHgF5/QxE+eizJ3uVXzx6nnMfwFaAR95rovvVdXrXSv6ItXzmP1KPY/578DlIvI9UAs8rapN9my3nsf8a+AlEZmJc+H4/qb8xU5E3sMJ82jPdY//DYTAz1P9foVz51AmUAY8cNH7bMJ/X8YYY7zAH5uGjDHGnAcLAmOMCXAWBMYYE+AsCIwxJsBZEBhjTICzIDCmAYnIOBH5b7frMOZkFgTGGBPgLAiMOQ0RmSwiG0Rkm4jMF5FmIlIiIs96xvlfIiLtPesO8Azw9p2ILBSRtp7Xk0QkTUS2i8gWEUn0bL6ViHwsIrtF5J0mPvKt8QMWBMacwjNMwR3Apao6AKeH7t1AOE5v1t7ACpwenwBvAr9R1X7AjpNefwdnSOj+wCjgp2EABgIzcMbO7w5c6vODMuYs/G6ICWO8YCIwGNjo+bLeEigA6oAPPOu8DXwiIpFAG1Vd4Xn9DZxhPCKAWFVdCKCqFQCe7W1Q1TzP821APLDa94dlzOlZEBjzSwK8oaq/+6cXRZ45Zb0LHZ/l5JFfa7H/h8Zl1jRkzC8tAW4VkRgAEWnnmeM5CGe0WoC7gNWqWgQcF5ExntfvAVaoajGQJyI3erbRQkTCGvQojKkn+yZizClU9XsR+T2wyDPBTTXwOFAKDPMsK8C5jgDOkObzPB/02fxjNMh7gPmekTKrgdsa8DCMqTcbfdSYehKRElVt5XYdxnibNQ0ZY0yAszMCY4wJcHZGYIwxAc6CwBhjApwFgTHGBDgLAmOMCXAWBMYYE+D+f6Fm5snOQJK1AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load Trained Model"
      ],
      "metadata": {
        "id": "MRZRy-ODkvOC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# load model -- 32.3 m\n",
        "model = load_model('/content/drive/MyDrive/Colab Notebooks/nwp-en-bsd/next_word_model.h5')"
      ],
      "metadata": {
        "id": "Apypo9qVjnOX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Predict next words"
      ],
      "metadata": {
        "id": "zU3RVvo0k0g9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def prepare_input(text):\n",
        "    words = tokenizer.tokenize(text)\n",
        "    x = np.zeros((1, WORD_LENGTH, len(unique_words)))\n",
        "    for t, word in enumerate(words):\n",
        "        x[0, t, word2id[word]] = 1.\n",
        "        \n",
        "    return x"
      ],
      "metadata": {
        "id": "zfCs0Z9VbxPI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def sample(preds, top_n=3):\n",
        "    preds = np.asarray(preds).astype('float64')\n",
        "    preds = np.log(preds)\n",
        "    exp_preds = np.exp(preds)\n",
        "    preds = exp_preds / np.sum(exp_preds)\n",
        "\n",
        "    return heapq.nlargest(top_n, range(len(preds)), preds.take)"
      ],
      "metadata": {
        "id": "-QOIx354fOQl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def predict_completion(text):\n",
        "    original_text = text\n",
        "    generated = text\n",
        "    completion = ''\n",
        "    while True:\n",
        "        x = prepare_input(text)\n",
        "        preds = model.predict(x, verbose=0)[0]\n",
        "        next_index = sample(preds, top_n=1)[0]\n",
        "        next_word = id2word[next_index]\n",
        "        text = text[1:] + next_word\n",
        "        completion += next_word\n",
        "        \n",
        "        if len(original_text + completion) + 2 > len(original_text) and next_word == ' ':\n",
        "            return completion"
      ],
      "metadata": {
        "id": "YoQssJddf5uU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def predict_next_word(text, n=3):\n",
        "    x = prepare_input(text)\n",
        "    preds = model.predict(x, verbose=0)[0]\n",
        "    next_indices = sample(preds, n)\n",
        "    return [id2word[idx] for idx in next_indices]"
      ],
      "metadata": {
        "id": "pPO74Nvpha0B"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Example #1"
      ],
      "metadata": {
        "id": "iuXyA-u84tVs"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Starting sentence: \"I look forward to working\""
      ],
      "metadata": {
        "id": "mutUyMdVl-7e"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# current model only accept input with 5 words\n",
        "input=\"I look forward to working\"\n",
        "print(predict_next_word(input, 5))\n",
        "\n",
        "# predicted sentence: \"I look forward to working with\" "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CfAJLlR0hj3k",
        "outputId": "80c49268-872b-44d2-e063-22334b3660bb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['with', 'on', 'about', 'it', 'in']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "input=\"look forward to working with\"\n",
        "print(predict_next_word(input, 5))\n",
        "\n",
        "# predicted sentence: \"I look forward to working with you\" "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z4D-lim2lopa",
        "outputId": "ae72625a-4f30-45bc-b0ee-2a42be55ec84"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['you', 'the', 'it', 'your', 'this']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "input=\"forward to working with you\"\n",
        "print(predict_next_word(input, 5))\n",
        "\n",
        "# predicted sentence: \"I look forward to working with you about\" "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uL0D_nBMlq2B",
        "outputId": "08b6c907-1cec-4cf1-c9a5-643e883faed4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['about', 'Thank', 'know', 'Yes', 'too']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "input=\"to working with you about\"\n",
        "print(predict_next_word(input, 5))\n",
        "\n",
        "# predicted sentence: \"I look forward to working with you about the\" "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fF_pnMzLl0G1",
        "outputId": "de1c141b-55a5-44a9-ac69-12dca1302762"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['the', 'it', 'this', 'a', 'that']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "input=\"working with you about the\"\n",
        "print(predict_next_word(input, 5))\n",
        "\n",
        "# predicted sentence: \"I look forward to working with you about the new\""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ol85UKKLl34s",
        "outputId": "e1467802-ee46-4559-f627-a7f9df3bf491"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['new', 'the', 'general', 'other', 'this']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Example #2"
      ],
      "metadata": {
        "id": "SU7PFo3t4wKx"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Starting sentence: \"Good morning I would like\""
      ],
      "metadata": {
        "id": "Sv99zPGZ5MCu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# current model only accept input with 5 words\n",
        "input=\"Good morning I would like\"\n",
        "print(predict_next_word(input, 5))\n",
        "\n",
        "# predicted sentence: \"Good morning I would like to\" "
      ],
      "metadata": {
        "id": "EoO4tco6oJON",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "45dfe417-4f50-480a-9196-056bf51fdff3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['to', 'that', 'you', 'a', 'the']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "input=\"morning I would like to\"\n",
        "print(predict_next_word(input, 5))\n",
        "\n",
        "# predicted sentence: \"Good morning I would like to be\" "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P8evw3TK417x",
        "outputId": "2367842b-3905-46a2-a6ac-63461e349e00"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['be', 'have', 'take', 'talk', 'know']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "input=\"I would like to be\"\n",
        "print(predict_next_word(input, 5))\n",
        "\n",
        "# predicted sentence: \"Good morning I would like to be the\" "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6GyuDW4m44TF",
        "outputId": "edcd09b5-0aab-483b-c797-284de9e503cb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['the', 'a', 'able', 'in', 'it']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "input=\"would like to be the\"\n",
        "print(predict_next_word(input, 5))\n",
        "\n",
        "# predicted sentence: \"Good morning I would like to be same\" "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AzyBFdNS4-2_",
        "outputId": "3cd4185f-9c3c-4d7c-8757-ae249da06374"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['same', 'general', 'other', 'new', 'for']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "input=\"like to be the same\"\n",
        "print(predict_next_word(input, 5))\n",
        "\n",
        "# predicted sentence: \"Good morning I would like to be same as\" "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0hVsRj9J5AwP",
        "outputId": "c12febfa-a099-4b80-da70-bf735a2d240a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['as', 'time', 'of', 'for', 'so']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "32V_dfiq5Cxu"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}