Seq2Seq_Language_Translation_model,_word_model_with_attention.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "id": "9064fc14",
      "metadata": {
        "id": "9064fc14"
      },
      "source": [
        "# Seq2Seq Language Translation model, word model with attention\n",
        "\n",
        "### Rosie Nguyen"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "61702b59",
      "metadata": {
        "id": "61702b59"
      },
      "source": [
        "### See Datasets: http://www.manythings.org/anki/"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "28064d99",
      "metadata": {
        "id": "28064d99"
      },
      "source": [
        "### Import Libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "620a1282",
      "metadata": {
        "id": "620a1282",
        "outputId": "81f4f75c-c382-4566-e78f-bc239b325c0e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tensorflow_text in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (2.6.0)\n",
            "Requirement already satisfied: tensorflow<2.7,>=2.6.0 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow_text) (2.6.0)\n",
            "Requirement already satisfied: tensorflow-hub>=0.8.0 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow_text) (0.12.0)\n",
            "Requirement already satisfied: h5py~=3.1.0 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (3.1.0)\n",
            "Requirement already satisfied: clang~=5.0 in c:\\users\\nhung\\appdata\\roaming\\python\\python38\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (5.0)\n",
            "Requirement already satisfied: numpy~=1.19.2 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (1.19.5)\n",
            "Requirement already satisfied: keras~=2.6 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (2.6.0)\n",
            "Requirement already satisfied: absl-py~=0.10 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (0.13.0)\n",
            "Requirement already satisfied: google-pasta~=0.2 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (0.2.0)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (3.17.2)\n",
            "Requirement already satisfied: astunparse~=1.6.3 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (1.6.3)\n",
            "Requirement already satisfied: typing-extensions~=3.7.4 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (3.7.4.3)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.37.0 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (1.41.1)\n",
            "Requirement already satisfied: keras-preprocessing~=1.1.2 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (1.1.2)\n",
            "Requirement already satisfied: tensorflow-estimator~=2.6 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (2.6.0)\n",
            "Requirement already satisfied: wrapt~=1.12.1 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (1.12.1)\n",
            "Requirement already satisfied: opt-einsum~=3.3.0 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (3.3.0)\n",
            "Requirement already satisfied: six~=1.15.0 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (1.15.0)\n",
            "Requirement already satisfied: termcolor~=1.1.0 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (1.1.0)\n",
            "Requirement already satisfied: gast==0.4.0 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (0.4.0)\n",
            "Requirement already satisfied: tensorboard~=2.6 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (2.7.0)\n",
            "Requirement already satisfied: flatbuffers~=1.12.0 in c:\\users\\nhung\\appdata\\roaming\\python\\python38\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (1.12)\n",
            "Requirement already satisfied: wheel~=0.35 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorflow<2.7,>=2.6.0->tensorflow_text) (0.37.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (3.3.4)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (1.6.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (0.6.1)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (2.26.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (2.0.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (0.4.4)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (1.33.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (58.0.4)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (4.2.2)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (4.7.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (1.3.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (0.4.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (2021.10.8)\n",
            "Requirement already satisfied: idna<4,>=2.5 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (3.2)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (2.0.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (1.26.7)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in c:\\anaconda3\\envs\\tensorflow\\lib\\site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow<2.7,>=2.6.0->tensorflow_text) (3.1.1)\n",
            "Note: you may need to restart the kernel to use updated packages.\n"
          ]
        }
      ],
      "source": [
        "pip install tensorflow_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "00108fd1",
      "metadata": {
        "id": "00108fd1"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "import string\n",
        "from string import digits\n",
        "import re\n",
        "from sklearn.utils import shuffle\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.layers import LSTM, Input, Dense,Embedding\n",
        "from keras.models import Model,load_model\n",
        "from keras.preprocessing.text import one_hot\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.models import model_from_json\n",
        "import pickle as pkl\n",
        "import numpy as np\n",
        "from keras.utils.vis_utils import plot_model\n",
        "import matplotlib.pyplot as plt\n",
        "import typing\n",
        "from typing import Any, Tuple\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.layers.experimental import preprocessing\n",
        "import tensorflow_text as tf_text\n",
        "import matplotlib.ticker as ticker"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "38f87193",
      "metadata": {
        "id": "38f87193"
      },
      "source": [
        "We will need to build a few layers from scratch, use this variable to switch between the custom and builtin implementations."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "7b7d9aeb",
      "metadata": {
        "id": "7b7d9aeb"
      },
      "outputs": [],
      "source": [
        "use_builtins = True"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "5a07ea6b",
      "metadata": {
        "id": "5a07ea6b"
      },
      "source": [
        "the shape checker class is used to check shapes throughout the tutorial."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "cf6ad2a4",
      "metadata": {
        "id": "cf6ad2a4"
      },
      "outputs": [],
      "source": [
        "#Shape checker\n",
        "class ShapeChecker():\n",
        "  def __init__(self):\n",
        "    # Keep a cache of every axis-name seen\n",
        "    self.shapes = {}\n",
        "\n",
        "  def __call__(self, tensor, names, broadcast=False):\n",
        "    if not tf.executing_eagerly():\n",
        "      return\n",
        "\n",
        "    if isinstance(names, str):\n",
        "      names = (names,)\n",
        "\n",
        "    shape = tf.shape(tensor)\n",
        "    rank = tf.rank(tensor)\n",
        "\n",
        "    if rank != len(names):\n",
        "      raise ValueError(f'Rank mismatch:\\n'\n",
        "                       f'    found {rank}: {shape.numpy()}\\n'\n",
        "                       f'    expected {len(names)}: {names}\\n')\n",
        "\n",
        "    for i, name in enumerate(names):\n",
        "      if isinstance(name, int):\n",
        "        old_dim = name\n",
        "      else:\n",
        "        old_dim = self.shapes.get(name, None)\n",
        "      new_dim = shape[i]\n",
        "\n",
        "      if (broadcast and new_dim == 1):\n",
        "        continue\n",
        "\n",
        "      if old_dim is None:\n",
        "        # If the axis name is new, add its length to the cache.\n",
        "        self.shapes[name] = new_dim\n",
        "        continue\n",
        "\n",
        "      if new_dim != old_dim:\n",
        "        raise ValueError(f\"Shape mismatch for dimension: '{name}'\\n\"\n",
        "                         f\"    found: {new_dim}\\n\"\n",
        "                         f\"    expected: {old_dim}\\n\")"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "e96b30e1",
      "metadata": {
        "id": "e96b30e1"
      },
      "source": [
        "### Load the data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "38e67759",
      "metadata": {
        "id": "38e67759",
        "outputId": "cc995dea-91fc-422b-b998-2577d10e7116"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>English</th>\n",
              "      <th>Vietnamese</th>\n",
              "      <th>Others</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Run!</td>\n",
              "      <td>Chạy!</td>\n",
              "      <td>CC-BY 2.0 (France) Attribution: tatoeba.org #9...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Help!</td>\n",
              "      <td>Giúp tôi với!</td>\n",
              "      <td>CC-BY 2.0 (France) Attribution: tatoeba.org #4...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Go on.</td>\n",
              "      <td>Tiếp tục đi.</td>\n",
              "      <td>CC-BY 2.0 (France) Attribution: tatoeba.org #2...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Hello!</td>\n",
              "      <td>Chào bạn.</td>\n",
              "      <td>CC-BY 2.0 (France) Attribution: tatoeba.org #3...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Hurry!</td>\n",
              "      <td>Nhanh lên nào!</td>\n",
              "      <td>CC-BY 2.0 (France) Attribution: tatoeba.org #1...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  English      Vietnamese                                             Others\n",
              "0    Run!           Chạy!  CC-BY 2.0 (France) Attribution: tatoeba.org #9...\n",
              "1   Help!   Giúp tôi với!  CC-BY 2.0 (France) Attribution: tatoeba.org #4...\n",
              "2  Go on.    Tiếp tục đi.  CC-BY 2.0 (France) Attribution: tatoeba.org #2...\n",
              "3  Hello!       Chào bạn.  CC-BY 2.0 (France) Attribution: tatoeba.org #3...\n",
              "4  Hurry!  Nhanh lên nào!  CC-BY 2.0 (France) Attribution: tatoeba.org #1..."
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#load raw text data file.\n",
        "df=pd.read_csv('vie.txt',delimiter=\"\\t\", header=None)\n",
        "#name the columns\n",
        "df.columns = ['English', 'Vietnamese', 'Others']\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "eb6d283a",
      "metadata": {
        "id": "eb6d283a"
      },
      "source": [
        "### Text Preprocessing "
      ]
    },
    {
      "cell_type": "markdown",
      "id": "abfc1427",
      "metadata": {
        "id": "abfc1427"
      },
      "source": [
        "**Remove unnecessary columns, shuffle rows but keep the correlatio between English and Vietnamese columns**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e2993130",
      "metadata": {
        "id": "e2993130",
        "outputId": "5a239e46-af89-4ed0-81e4-4d0423d3baec"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>English</th>\n",
              "      <th>Vietnamese</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>6626</th>\n",
              "      <td>There are times when we have to hide the truth.</td>\n",
              "      <td>Có những lúc mà chúng ta phải che giấu đi sự t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2967</th>\n",
              "      <td>When was printing invented?</td>\n",
              "      <td>Việc in ấn ra đời khi nào?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>199</th>\n",
              "      <td>I play tennis.</td>\n",
              "      <td>Tôi chơi môn quần vợt.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3413</th>\n",
              "      <td>She is a friend of my wife's.</td>\n",
              "      <td>Nó là bạn của vợ tao.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3633</th>\n",
              "      <td>I was amazed to learn I'd won.</td>\n",
              "      <td>Tôi đã ngạc nhiên khi biết được là tôi đã thắng.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                              English  \\\n",
              "6626  There are times when we have to hide the truth.   \n",
              "2967                      When was printing invented?   \n",
              "199                                    I play tennis.   \n",
              "3413                    She is a friend of my wife's.   \n",
              "3633                   I was amazed to learn I'd won.   \n",
              "\n",
              "                                             Vietnamese  \n",
              "6626  Có những lúc mà chúng ta phải che giấu đi sự t...  \n",
              "2967                         Việc in ấn ra đời khi nào?  \n",
              "199                              Tôi chơi môn quần vợt.  \n",
              "3413                              Nó là bạn của vợ tao.  \n",
              "3633   Tôi đã ngạc nhiên khi biết được là tôi đã thắng.  "
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#remove column others\n",
        "df= df.drop(columns= 'Others') \n",
        "#shuffle the rows but still keep the correct translation orders\n",
        "df= df.reindex(np.random.permutation(df.index))\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "8dcbf9c8",
      "metadata": {
        "id": "8dcbf9c8"
      },
      "source": [
        "**Text processing: convert to lowercase, remove punctuation, remove whitespaces at the start and end of sentences**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8e0ae0fc",
      "metadata": {
        "id": "8e0ae0fc",
        "outputId": "1d7092b8-78d0-470c-8f5e-910edea3b24f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Tôi không học được gì từ giáo viên đó cả.\n",
            "I learned nothing from the teacher.\n"
          ]
        }
      ],
      "source": [
        "#text preprocessing\n",
        "targ = df['English'].values\n",
        "inp = df['Vietnamese'].values\n",
        "print(inp[-1])\n",
        "print(targ[-1])"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "bcd89531",
      "metadata": {
        "id": "bcd89531"
      },
      "source": [
        "**Create a tf.data dataset**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d7dc970b",
      "metadata": {
        "id": "d7dc970b"
      },
      "outputs": [],
      "source": [
        "BUFFER_SIZE = len(inp)\n",
        "BATCH_SIZE = 64\n",
        "\n",
        "dataset = tf.data.Dataset.from_tensor_slices((inp, targ)).shuffle(BUFFER_SIZE)\n",
        "dataset = dataset.batch(BATCH_SIZE)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "84e0d9a4",
      "metadata": {
        "id": "84e0d9a4",
        "outputId": "be1b6b08-f95c-4e7d-9eb3-9c0d76c43c54"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tf.Tensor(\n",
            "[b'T\\xc3\\xb4i c\\xe1\\xba\\xa7n th\\xc3\\xaam n\\xe1\\xbb\\xafa.'\n",
            " b'T\\xc3\\xb4i \\xc4\\x91\\xc3\\xa3 gi\\xc3\\xbap Tom s\\xc6\\xa1n c\\xc3\\xa1i h\\xc3\\xa0ng r\\xc3\\xa0o \\xc4\\x91\\xc3\\xb3.'\n",
            " b'T\\xc3\\xb4i nghi\\xc3\\xaan c\\xe1\\xbb\\xa9u nhi\\xe1\\xbb\\x81u gi\\xe1\\xbb\\x9d li\\xe1\\xbb\\x81n v\\xc3\\xa0o bu\\xe1\\xbb\\x95i t\\xe1\\xbb\\x91i.'\n",
            " b'\\xc4\\x90\\xe1\\xbb\\xabng ngh\\xc4\\xa9 l\\xc3\\xa0 t\\xc3\\xb4i \\xc4\\x91\\xc3\\xa3 kh\\xc3\\xb4ng th\\xe1\\xbb\\xad.'\n",
            " b'B\\xe1\\xba\\xa1n ngh\\xc4\\xa9 \\xc4\\x91i\\xe1\\xbb\\x81u g\\xc3\\xac \\xc4\\x91\\xc3\\xa3 g\\xc3\\xa2y ra m\\xc3\\xa2u thu\\xe1\\xba\\xabn trong ho\\xe1\\xba\\xa1t \\xc4\\x91\\xe1\\xbb\\x99ng th\\xc6\\xb0\\xc6\\xa1ng m\\xe1\\xba\\xa1i gi\\xe1\\xbb\\xafa Nh\\xe1\\xba\\xadt B\\xe1\\xba\\xa3n v\\xc3\\xa0 Hoa K\\xe1\\xbb\\xb3?'], shape=(5,), dtype=string)\n",
            "\n",
            "tf.Tensor(\n",
            "[b'I need more.' b'I helped Tom paint the fence.'\n",
            " b'I study for many hours at night.' b\"Don't think I didn't try.\"\n",
            " b'What do you think has caused the present trade friction between Japan and the U.S.?'], shape=(5,), dtype=string)\n"
          ]
        }
      ],
      "source": [
        "for example_input_batch, example_target_batch in dataset.take(1):\n",
        "  print(example_input_batch[:5])\n",
        "  print()\n",
        "  print(example_target_batch[:5])\n",
        "  break"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "4c2fcc40",
      "metadata": {
        "id": "4c2fcc40"
      },
      "source": [
        "**Standardization**"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "c0df71a3",
      "metadata": {
        "id": "c0df71a3"
      },
      "source": [
        "The model is dealing with multilingual text with a limited vocabulary. So it will be important to standardize the input text.\n",
        "The first step is Unicode normalization to split accented characters and replace compatibility characters with their ASCII equivalents.\n",
        "The tensroflow_text package contains a unicode normalize operation:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e3c4eeda",
      "metadata": {
        "id": "e3c4eeda",
        "outputId": "5b0a57f1-6ae8-4be5-fc12-3d50c55d9424"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "b'B\\xe1\\xbb\\x99 phim n\\xc3\\xa0y \\xc4\\x91\\xc6\\xb0\\xe1\\xbb\\xa3c d\\xe1\\xbb\\xb1a tr\\xc3\\xaan m\\xe1\\xbb\\x99t cu\\xe1\\xbb\\x91n ti\\xe1\\xbb\\x83u thuy\\xe1\\xba\\xbft'\n",
            "b'Bo\\xcc\\xa3\\xcc\\x82 phim na\\xcc\\x80y \\xc4\\x91u\\xcc\\x9bo\\xcc\\x9b\\xcc\\xa3c du\\xcc\\x9b\\xcc\\xa3a tre\\xcc\\x82n mo\\xcc\\xa3\\xcc\\x82t cuo\\xcc\\x82\\xcc\\x81n tie\\xcc\\x82\\xcc\\x89u thuye\\xcc\\x82\\xcc\\x81t'\n"
          ]
        }
      ],
      "source": [
        "example_text = tf.constant('Bộ phim này được dựa trên một cuốn tiểu thuyết')\n",
        "\n",
        "print(example_text.numpy())\n",
        "print(tf_text.normalize_utf8(example_text, 'NFKD').numpy())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "768ec816",
      "metadata": {
        "id": "768ec816"
      },
      "outputs": [],
      "source": [
        "def tf_lower_and_split_punct(text):\n",
        "  # Split accecented characters.\n",
        "  text = tf_text.normalize_utf8(text, 'NFKD')\n",
        "  text = tf.strings.lower(text)\n",
        "  # Keep space, a to z, and select punctuation.\n",
        "  text = tf.strings.regex_replace(text, '[^ a-zđ.?!,]', '')\n",
        "  # Add spaces around punctuation.\n",
        "  text = tf.strings.regex_replace(text, '[.?!,]', r' \\0 ')\n",
        "  # Strip whitespace.\n",
        "  text = tf.strings.strip(text)\n",
        "\n",
        "  text = tf.strings.join(['[START]', text, '[END]'], separator=' ')\n",
        "  return text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3c9e7ca2",
      "metadata": {
        "id": "3c9e7ca2",
        "outputId": "634aff41-1111-46d5-ea8b-218da4a6c88c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Bộ phim này được dựa trên một cuốn tiểu thuyết\n",
            "[START] bo phim nay đuoc dua tren mot cuon tieu thuyet [END]\n"
          ]
        }
      ],
      "source": [
        "print(example_text.numpy().decode())\n",
        "print(tf_lower_and_split_punct(example_text).numpy().decode())"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "1d280809",
      "metadata": {
        "id": "1d280809"
      },
      "source": [
        "**Text Vectorization**"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "8afa04e8",
      "metadata": {
        "id": "8afa04e8"
      },
      "source": [
        "TextVectorization layer will handle the vocabulary extraction and conversion of input text to sequences of tokens."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "14e4184a",
      "metadata": {
        "id": "14e4184a"
      },
      "outputs": [],
      "source": [
        "max_vocab_size = 5000\n",
        "\n",
        "input_text_processor = preprocessing.TextVectorization(\n",
        "    standardize=tf_lower_and_split_punct,\n",
        "    max_tokens=max_vocab_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9740492e",
      "metadata": {
        "id": "9740492e",
        "outputId": "99f3418d-8f1c-498c-90f6-636af7a66302"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['', '[UNK]', '[START]', '[END]', '.', 'toi', 'khong', 'co', 'ban', 'tom']"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#vietnamsese vocabulary \n",
        "input_text_processor.adapt(inp)\n",
        "\n",
        "# the first 10 words from the vocabulary:\n",
        "input_text_processor.get_vocabulary()[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f86cdf62",
      "metadata": {
        "id": "f86cdf62",
        "outputId": "1ce72261-4791-497c-fcec-8ee8d25292eb"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['', '[UNK]', '[START]', '[END]', '.', 'i', 'to', 'the', 'tom', 'you']"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#english vocabulary \n",
        "output_text_processor = preprocessing.TextVectorization(\n",
        "    standardize=tf_lower_and_split_punct,\n",
        "    max_tokens=max_vocab_size)\n",
        "\n",
        "output_text_processor.adapt(targ)\n",
        "output_text_processor.get_vocabulary()[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4794368e",
      "metadata": {
        "id": "4794368e",
        "outputId": "edb9ee35-960e-4831-906a-5b4f324e59bc"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(3, 10), dtype=int64, numpy=\n",
              "array([[  2,   5,  79, 203, 116,   4,   3,   0,   0,   0],\n",
              "       [  2,   5,  11, 136,   9, 609,  34, 215, 592,  16],\n",
              "       [  2,   5, 576, 361,  67,  61, 430,  50, 268,   5]], dtype=int64)>"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#convert a batch of strings into a batch of token IDs:\n",
        "example_tokens = input_text_processor(example_input_batch)\n",
        "example_tokens[:3, :10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d12a5006",
      "metadata": {
        "id": "d12a5006",
        "outputId": "e231f222-ba5b-407d-e60d-794296706a4a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'[START] toi can them nua . [END]                '"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#convert token IDs back to text\n",
        "input_vocab = np.array(input_text_processor.get_vocabulary())\n",
        "tokens = input_vocab[example_tokens[0].numpy()]\n",
        "' '.join(tokens)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f7ffd56e",
      "metadata": {
        "id": "f7ffd56e",
        "outputId": "07d7e12d-34f8-4a7f-a533-6efe2f4a7cb4"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Mask')"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAiZUlEQVR4nO3de5zdVXnv8c8zk0kmkBsJSQiThAQJIVzkYgzUO+KtYA+xtbRKlWPxRHuo1R7b4u3gBe3B9lSw1VapgvEKFLGghyMiBdGq3DFcEggJJOQeQkLul5n99I+9I0NYz28me2bv+a2Z7/v1mteevdas/VuzZ83aaz97XczdERGR/LQMdAVERKQ+6sBFRDKlDlxEJFPqwEVEMqUOXEQkU+rARUQypQ68gczsdWa2aqDrIZIbM7vDzN470PUoO3XgvWRm27t9VcxsV7f75w9w3X7b2GsvGpVudVtlZteZ2csHso4y+JjZU2a218wOPyD9ATNzM5sxQFUbMtSB95K7j9r/BawEfq9b2ncGun4HWFOr52jgDGAJ8HMzO2tgqyWD0JPAO/bfMbOTgEMGrjpDizrwPjKzEWZ2hZmtqX1dYWYjgp/9CzN71Mym1sr9XzNbaWbrzewrZjay9nOvq42cP2xmG8xsrZm952Dr5lWr3P0S4GvA52uPb2Z2ee2xt5rZQ2Z2Yl+eBxmyvgW8u9v9C4Bv7r9jZufURuRbzexpM/tUt7x2M/u2mW0ysy1mdo+ZTT7wAmY2xcwWmdlfN/IXyZE68L77ONVR7inAycA84BMH/pCZXQL8d+C17r4KuAw4tlbuGKADuKRbkSOAsbX0C4Evm9lhfajnDcBpZnYo8CbgNbXrjwXOAzb14bFl6Po1MMbM5phZK/DHwLe75e+g2sGPA84B/szM5tfyLqDa/qYBE4D3A7u6P7iZzQR+BnzJ3f++cb9GntSB9935wGfcfYO7bwQ+DbyrW76Z2ReodppnuvtGMzNgAfCX7v6su28D/pZq499vX+1x97n7zcB2YHYf6rkGMKr/SPuohleOA8zdF7v72j48tgxt+0fhbwQWA6v3Z7j7He7+kLtX3H0R8D3gtbXsfVQ77mPcvcvd73P3rd0e93jgduCT7n5lM36R3Awb6AoMAkcCK7rdX1FL228c1c76j9z9uVraRKpxwvuqfTlQ7Vxbu5Xb5O6d3e7vBEb1oZ4dgANb3P0/zOxLwJeBo8zsBuCvDvjnEemtbwF3AjPpFj4BMLPTqb7bPBEYDowA/q1buWnANWY2jurI/ePuvq+Wfz7wBHB9g+ufLY3A+24NcFS3+9NrafttBt4KXG1mr6ylPUP1reIJ7j6u9jW29sFjo7wNuN/ddwC4+z+6+8uojnKOBRRflLq4+wqqH2aeTTVU1913gZuAae4+FvgK1cEKtXeXn3b344FXUP0/6R5P/xTV/5Xv1sIzcgB14H33PeATZjaxNp3qEl4YA8Td76A6mrjBzOa5ewX4V+ByM5sEYGYdZvbm/qxY7cPKDjP7JPBe4GO19Jeb2elm1kY1RrkbqPTntWXIuRB4/f4BQjejgWfdfbeZzQPeuT/DzM40s5NqnfNWqiGV7u1wH/CHwKHAN81M/dUB9IT03WeBe4FFwEPA/bW0F3D3W4E/BX5oZqcBF1N9e/hrM9sK/JS+xbi7O9LMtlONm98DnAS8zt1/UssfQ/UFZDPVkM8mQB8QSd3cfZm735vI+p/AZ8xsG9XBzXXd8o6gGh7ZSjV2/jOqYZXuj7sX+H1gMnCVOvEXMh3oICKSJ72aiYhkSh24iEim1IGLiGRKHbiISKaaupBnuI3wdg49qDLWEr/G7J04MpnetmHnQV3jt9dqSz8dvndfMl3KZRubn3H3ic2+7uHjW33GtLZmX3bAPL5Ie1U1W9S2m9qBt3Mop6c2xCuYGdQ6Ku7wV73rpGT6kV+856DrBtA65UX76ADQ+fSaZDoArunTZfFTv35Fzz/V/2ZMa+PuW6YPxKUHxJuPPHmgqzDkRG1bIRQRkUypAxcRyVQpNrOyFuv5hxI6/vmBZHrL2DFhmd2nzYwf8PbfJJNXffyMsMj0v0stPqMwLLT2urgOk+c/ls5QqEZK4pY16f+T/qZQTc80AhcRyZQ6cBGRTJUihFJk/Z+kZ5oAHPHD9KQD337ghmjPawvCJAA/XpkOh5z+0blhmcq+znRGQchj8rmLwzyRnCns0VwagYuIZEoduIhIptSBi4hkqlcx8Np5dV+jeq6dUz2Y4DHgWmAG8BRwnrtvrqcSj1/x8jBv9tXPhXmVZ9OXq+zaHV+sIDYdxe/G8av48SRrjW7bZaC49ODV2xH4F4Efu/txwMlUT8/4CHCbu88CbqvdF8mN2rZkq8cO3MzGAq8Bvg7VI47cfQtwLrCw9mMLgfmNqaJIY6htS+56E0KZCWykeqr6ycB9wAeBye6+tvYz66ieWfciZrYAWADQTnoXszn/J96DaPfxHWHervlByOPmeJreksuODfMYkQ6vzLn0mbBIZVXBRleBJz73sjDv2C+tSmd0xaGfxR+bmkyfdVHBpl5a2Ql9aNvd2/X0jnLPxm3WysmyG4yhpN6EUIYBpwH/4u6nUj3F/AVvKb16sGbycE13v9Ld57r73DZG9LW+Iv2p7rbdvV1PnNDalMqKHKg3HfgqYJW731W7fz3VRr/ezKYA1G43NKaKIg2jti1Z6/G9n7uvM7OnzWy2uz8GnAU8Wvu6ALisdntjvZXoXLs+zGtbH//vjAg2rdr4B8eHZeZc+lRckZHt6fSCWS02fHgyfd9ps8Iyx3xva5jXuXJ1+jrzTgzLzLyhK5k+bPZL4us8tizMGyrhlWa0bSmPolBSruGV3gbvPgB8x8yGA8uB91AdvV9nZhcCK4DzGlNFkYZS25Zs9aoDd/cHgdSGIInjdUTyobYtOdNKTBGRTJVi/tOyy+eFeWOWxq8xk75yVzL9sMd3hWWi1ZsAnfPmJNOHL9kelunakT5AueXO9GETAPVEmP2uRWFe9EcM9kkUaZhcY8m50ghcRCRT6sBFRDJVihDKrIsfDPN2/G78lsyDqXXDNqfDGgDv/k1w5iRw9ew96esUnLEZnefplYLXxiEyTU+GnmiqnkIrjaERuIhIptSBi4hkSh24iEimShEDr+zdG+aNvDHeUW/YUdOS6TvnTArLXD17RpjX9fpT0xm70kvVAbZPTy+/H3NjvGy3sjsdawdoDeLtNnFCWKbziSfTjzXq0LCM790X5rWMHZ2+zsZNYRnF9aXIYNwRsQxxfY3ARUQypQ5cRCRTpQih1Ms3b0mmj/x5/FZ/yVWnhXkdP0zv63zIf8SrKkf9Mh06qDeg0LVlSzojSi96rG3b6qpDZUPBmaIy5JUhdCBVGoGLiGRKHbiISKZKEUKJVjMCeCV5UltVJR2o2Dc3PvdyzifiMyw716yLKhHXQWQQUpgkDxqBi4hkSh24iEim1IGLiGSqFDHwzX8SH+gw7pvpQxsALFi1uPJNweHEwEsejHcqtNb0NMLt8wsOnLjlkWR61/YdYZnCFZKd6WMYWqYeGZbZcVx6lebKc8IiHHvRfWFedFBzyyGHxA942Ngwa/ll6d93+h+mnztAnzsMsMG4crLs6vncQSNwEZFMqQMXEclUKUIoh3377jizYIph59TDk+lHf+resMzeV58UXyt4137o9+P6ddXxVr+eFZKVpcvCvBFB3qwfxo9XMDkT35U+U7QSpAOwKV79Ov3tBReT7GiKYXloBC4ikil14CIimVIHLiKSqV7FwM3sKWAb0AV0uvtcMxsPXAvMAJ4CznP3zfVUwoa1hXnbz4njbaN+lN4l0LviAxiWnx/H1GddeH86Q1PaBq1Gt+1cKc6dh4MZgZ/p7qe4+9za/Y8At7n7LOC22n2RHKltS5b6EkI5F1hY+34hML/PtREpB7VtyUJvpxE68BMzc+Cr7n4lMNnd19by1wGTUwXNbAGwAKCd9Eq+ojMxxyzaGNdqWkf68Z5eHRaZ83fPxY939FHp9Lb4aep8LJjep7BLLupq293b9fSOUszG7Vf1rsRU6KW5etvyXuXuq81sEnCrmS3pnunuXvsHeJHaP8SVAGNsfNH0Y5GBUFfb7t6u557crnYtA6JXIRR3X1273QD8AJgHrDezKQC12w2NqqRIo6htS856HIGb2aFAi7tvq33/JuAzwE3ABcBltdsb661E4YEOz8Yf/tvoUcn0ltGjwzKVFavia+1LbyTlnfvCMpKvZrTtXCkUkofehFAmAz8ws/0//113/7GZ3QNcZ2YXAiuA8xpXTZGGUNuWrPXYgbv7cuBFL8fuvgk4qxGVEmkGtW3JnVZiiohkqvTzn3zX7jCva3N6SmBL+4iwTMuk9A6GAJ0rg+mHVvA6p+mCMggVTSNUfLw8NAIXEcmUOnARkUyVIoQSnUUJ0Hn6nDBv75h09UfenN7kCmDbafHZkqN370mmr7jwmLjMivQajrHXxodKaFqi5KyeVZoKuzSGRuAiIplSBy4ikil14CIimSpFDLzoAIbnjo6nBI5alY4lb7lpRlhmw7r4WrP+/ZlkesffHvxWGNrdSHKmmHUeNAIXEcmUOnARkUyVIoRSNI1wwjXxlCXfmw6h7J7wsrDMnJ+tiB/v0PSBE107doZlOOOkZPKwZWvCItvPmBnmtf8wmH6oFZ/SRPUe6NCfFMbpmUbgIiKZUgcuIpKpUoRQimahUInnc7Qcd3QyfewND4ZlugquteeNpyTT29fsCMv4vnRoo3PjprBM+00F53yKDEIKhzSGRuAiIplSBy4ikil14CIimSpHDLwgzg1xzNq2pw97WHJ5HG+bc1lwaAMw8ueL0xnD28IyleBQCU37k6FIse7m0ghcRCRT6sBFRDJVihBKkdYxo8O8rlXp1Y7HfmBVWMZHjQrzKrt2JdNbzMIyLcOHJ9Pt0JFhmegsT0ChF8maDntoLo3ARUQypQ5cRCRT6sBFRDLV6xi4mbUC9wKr3f2tZjYTuAaYANwHvMvd9/Z3Bbu2bgvzWtrThz3YiPgQiK1vOC7MG/ub9BJ3b4+nEbJsZTK5MM5doHV0OuZvY8eEZRZ/bGoyfdZF98QXUqwdGLh2Lc9r1s6HgzHWfjAj8A8C3SdKfx643N2PATYDF/ZnxUSaRO1astWrDtzMpgLnAF+r3Tfg9cD1tR9ZCMxvQP1EGkbtWnLX2xDKFcDfAPvf308Atrh7Z+3+KqAjVdDMFgALANpJH5hgLfE0PQum6QFwXPpgBH/kibDI6JvuD/P2zTs+mb53XFyHp/8mHZI59r2PhmVaDh8f5u18afJp5IRLF4Vlus5In9lpbfXNEo12hyz6W1R27yl4wNKGa66gH9r19I7Sz8YVmntIRbPCNT2OwM3srcAGd7+vngu4+5XuPtfd57YRx6ZFmqk/2/XECfGJUiKN1JuhwyuB/2ZmZwPtwBjgi8A4MxtWG61MBeJNRkTKR+1astdjB+7uHwU+CmBmrwP+yt3PN7N/A95O9RP7C4Ab661E5+tOiSt4x4NhXuumrcn0vWecEJYZviRepbnytekQz4zr0yEKgGPfm368lvHjwjKda9aFecNXp1eXLv3/YZFQ0RZh9fBgpWqOmtGupXcG4+yQZunLPPCLgf9lZk9QjR1+vX+qJDKg1K4lGwf16Yu73wHcUft+OTCv/6sk0lxq15IrrcQUEclUKeY/FcW5sfg1prL+mWT68IIpbZUt8QrJGVc/mc4omD636Z2nJtMP+8ZdYZkST6sTaQjFuRtDI3ARkUypAxcRyVQpQiith08I8zpnTgnzWh5els4YHR/a4DOOCPO67l+STLfWeKHGuKWHhXkiIo2kEbiISKbUgYuIZKoUIZTODenZJAAtBfuBdwWzTZZ9elxYZvbH0nt+A1SCzZ98b7wdtP1nsMmUZprIEKTZJs2lEbiISKbUgYuIZEoduIhIpkoRAy860IFKvKfesOBghGMvWh4/3L59va7Xfq3T0mdOAviY9A6GlUfjQyW88+DrIJKD6NAExcYbQyNwEZFMqQMXEclUKUIorR3xasvDr42nES7fOiaZvufayWGZHefEjzftjx5Lpm8888iwTLRpVeXV8VvG1mjqIWDD2tLps9PnfwLwZHBIRcEKUt8Tb/gVnX1pEwtWzE5K/y0A+JWmWg51zTyPssz6O5SkEbiISKbUgYuIZEoduIhIpkoRA/dnt4R5616xM8wbPSodQx3VFR8aPPFHh8b1GJXOq+dwhpY7H4iLxI+Gd3WlMxYtLijVz6LDi5+LD8MgnjUpg4ymBJaHRuAiIplSBy4ikqlShFDq1bV9RzK9pX1EWOaJD80K846+Ph0iaF0WBz1sQvpABy/YYTEMkwCd8+Yk04cvCaYKUrCbo6bpSQOUfUrgUArxaAQuIpIpdeAiIpkqRQhl7XfjzaKGtcZhgEkXBasJo1kUwMxL7gnzKvVsMlU0M6MOLT+7P5ne2a9XEakaSuGGwajHEbiZtZvZ3Wb2GzN7xMw+XUufaWZ3mdkTZnatmaXXX4uUlNq25K43IZQ9wOvd/WTgFOAtZnYG8Hngcnc/BtgMXNiwWoo0htq2ZK3HDtyrttfuttW+HHg9cH0tfSEwvxEVFGkUtW3JXa9i4GbWCtwHHAN8GVgGbHH3/aHZVUBHUHYBsACgnfThB1MujuPcldHxlEA8Hevu2rQ5LFI0xdA6jkrX4ek1YZnHvnBiOmNE/Dsd9xfpXQ8BbNLh6YydcVzfgzwbnt7ZEMCDA6EB/Nj082DL4qmMT/7l8WHe0f+Y/n07n43/Ts2aAllv2+7erqd3lOKjpLqUfUrgUFLP5xG9moXi7l3ufgowFZgHHNfbC7j7le4+193ntlHQGYsMgHrbdvd2PXFCvG2vSCMd1DRCd98C3A78DjDOzPYPPaYCq/u3aiLNo7YtOerxvZ+ZTQT2ufsWMxsJvJHqhzy3A28HrgEuAG6stxKVx5fFea94aVxwdXrTqj1vOCUssvG0OKxw1DXpUEll1+6wzKz33x3mReJ1mMC2+MCJprnv4YMuMv2TvwzzyjoFshltezDS1MPy6E3wbgqwsBYrbAGuc/cfmdmjwDVm9lngAeDrDaynSCOobUvWeuzA3X0RcGoifTnVmKFIltS2JXdaSi8ikqlSzH9qOT7eIfCJ98XljvlVOt0q8e6BU38Sx5g7n1yZztCufjLEKM6dB43ARUQypQ5cRCRTpQihdD38eJg3638ULP4JVhpunRnvPTT5gafierz8hGS63/NIXAeFV2QQKlqhqfBKeWgELiKSKXXgIiKZKkUIxVoszjsmvbESQGVxegXn4V+7K77YEZPDrJbH0rNQ9rzhtLDMiDVbgwcreG30eJaMLwtmwlj8HEUrRa013qPD6zm8QoR8N8AajKEfjcBFRDKlDlxEJFPqwEVEMlWKGHjr5Elh3uPvPizMm/WNmcn09a8JDkUA9oyN69Hx+V8n04fdem9YpnBnwQHmnZriKPUZjPHiwUgjcBGRTKkDFxHJVClCKIw+NMw69p+ejst1pUMEkxbGZzfaUVPDvEqwEpP7l4RlNB1PBiOtxMyDRuAiIplSBy4ikil14CIimSpFDLyysuDQ74LDGfzk9EEQ9syzYRl7dkv8eEueCTI0HU+GFsW586ARuIhIptSBi4hkqhwhlN17wryinQpbn0mfb7nnVcF0QGD4uvhMzHXvPCaZvm90WIRpl6ZXb2onQMlZf+84qJBMY2gELiKSKXXgIiKZKkUIZdjR8aENnctXhHmVteuT6bvmHRGWaXsm/pU7rkqffel794ZlGNmeTO467diwSOulG8O8lgvSIaNVb58eluk8JJ0+9XPp8A6gmTXSVGU4BGIwhnF6HIGb2TQzu93MHjWzR8zsg7X08WZ2q5ktrd3G2waKlJDatuSuNyGUTuDD7n48cAZwkZkdD3wEuM3dZwG31e6L5ERtW7LWYwfu7mvd/f7a99uAxUAHcC6wsPZjC4H5DaqjSEOobUvuDioGbmYzgFOBu4DJ7r62lrUOSJ4WbGYLgAUA7aSDtb5pc3jNUXfGhzNsf0165eS4XxTsYFjAW9OvZ2vfd2pYpuPf06tIK/+5KCxTOTOOP0c5R3wh3mFR+u5g23b3dj29oxQfJZXaYIw/l0GvZ6GY2Sjg+8CH3P0FR7G7uwPJNe/ufqW7z3X3uW2M6FNlRRqhnrbdvV1PnBDP+RdppF514GbWRrWBf8fdb6glrzezKbX8KcCGxlRRpHHUtiVnPb73MzMDvg4sdvcvdMu6CbgAuKx2e2O9leh6bmuYt/NN8ai95dQ5yfTKw0vjiw2Lf+UVf31KMn3GFenphQA+LT1lsWhqZDT9EaCya3dwIU3762/NaNtSVc80QoVdetab4N0rgXcBD5nZg7W0j1Ft3NeZ2YXACuC8htRQpHHUtiVrPXbg7v4LINqQ5Kz+rY5I86htS+60lF5EJFOlmP/UOibe7s/3Fezct3h5Mnn5p+aGRWZ8PF5ePu3Tv0ymd8U1gOeeK8oVKTXFmfOmEbiISKbUgYuIZKoUIZSurfEhC4UHOkwYn0w/5sr4jE0vCtcEB0tUinYj1PQ+yZim9+VNI3ARkUypAxcRyVQpQihFvJLcYqWaF6xajIMuYCOGh3lhKEdhEhliFCbJg0bgIiKZUgcuIpIpdeAiIpkqRQx8w43xAcCTLksfGgxQueuhZPru16Z3KQQYeWt80MKwiROS6VtfdXRYZsyjzybTOx9bFpax1nj/aO8sWHkq0iTNPIRY8fb6aQQuIpIpdeAiIpkqRQhl8tviAxhseDztzy39+jPyJw/GFys40KFz46Zk+ug7OsMyG34/Ha7Z+QcTwzIzvvJYmLfjd16STB/x/+4Ny2iao+RMq0HrpxG4iEim1IGLiGRKHbiISKZKEQMvWi6/6kOnhnlT//6e9ON1FRzBUHCttR8+I5k+5R/iQyDGfy19CER6n8SqOKIOI36UjsOLlIXiz+WhEbiISKbUgYuIZKoUIZSW9hFh3sQH4pWJrVOPTKb75i1hmWUXnxDmjXo6nT5sekdYZvX8acn0Ixc+GpahY1KYZevSIZSn/mx2WGbGt4KKF1h2YbreAEd9MggZabqi0NxVmmVQ5pCRRuAiIplSBy4ikqlyhFBGjwrz2u98JMzzl0xPpldWrQnLTLovDgMccsPdyfTOgtDB5C+mwxcF82Bgy5ai3KSpn03PdoHiWS2Roy45+LCLDB1lDhvI83ocgZvZVWa2wcwe7pY23sxuNbOltdvDGltNkf6nti25600I5RvAWw5I+whwm7vPAm6r3RfJzTdQ25aM9diBu/udwIGbXp8LLKx9vxCY37/VEmk8tW3JXb0x8Mnuvrb2/TpgcvSDZrYAWADQziHJn+nalD4UoSfrzkyvd9z63nFhmfYN8WvW6LFjkuk7Xx0fONF+833J9GiKI8CT746n8LVtT6dPfHBPWKb1jgeS6To4oi69atvd2/X0jlJ8lNSvBuNUwcEY1+/zLBR3dyBcn+7uV7r7XHef20Y831ukbIradvd2PXFC/EIp0kj1duDrzWwKQO12Q/9VSWRAqW1LNup973cTcAFwWe32xr5UYsv588K8CbetCPOO+Or9yfQjx6VDIQB0Fkzwm5CecNC6++BXIHauXB3mTbu0OVP4vFMrJ+vQr21bqgZj+KIMejON8HvAr4DZZrbKzC6k2rjfaGZLgTfU7otkRW1bctfjCNzd3xFkndXPdRFpKrVtyZ2W0ouIZKoU85/2jLUwb9cJ8XS8kY+lP/3vKlhK3zrr6LgiG9PTGdsX7QiL7HrDaem6LYrj3J3r4s/Fop0Pd7x0SljmkDsWJ9Nt0uFhGTrjBfg75xyRTB+5JpjjCOzqiLdD2HpUWzL98K/GB2Vo58PBZTBOS+xv9XxOoBG4iEim1IGLiGSqFCGUyf98V5jXOr5gL6FxY5PJLSNHhkUqBdP7rC39Vn/Cj+Kph/fdnC4zc1l61SlA64nxys7Ohx9Ppo9YEYdkwtpt2xaWKTI8uFbRDovDF8V5BYEckZCmHvZMI3ARkUypAxcRyVQpQiitU9KzHgB8y3Nx3vb07BDfF8+w8K44EDAsCNds+PDEsMy0X6VnUhQdAiEyWCns0VwagYuIZEoduIhIptSBi4hkqhQx8K6168K81nHjwjwbll6J6Tt2hmUqu+ODERb/7/QqyDmXPBWW2fie05PpJ73vobDM2jfHT/uSz6enGB73T/FnAbuPHJ1Mt4IwvF+8Mcwb+YHhyfSupcvDMkUrXHfOHJfOaIlX4I64JX1IhQ6iKDetuGyM1mAhtkbgIiKZUgcuIpKpcoRQXvnSMK/1kZVxuU2bk+lFUwWLzomc9f570xnT4g21Drs6vYp01VX1TSOc9b67k+lFqyDbHq7jQj+Ns+JJmAVlliwN84YvOfjHC8/ok1LTNMJGSf9/aQQuIpIpdeAiIplSBy4ikqlSxMBbfh5PPdp2y1Fh3jO/mJ1Mn3lVfBAylTg2veTi9LWO+9yysMyzf3pGMn38VQWHFRTREnzJWD3TCBU3r59G4CIimVIHLiKSqVKEUIoccvaqMG/6y9IrEItWdo68PT5eYPbvpc+W3HrWcWGZ8eenpzm2fCt+aisFuyWKiPSWRuAiIplSBy4ikqnSh1Bap8arILdPbk+mH3JienYKwM7XFiwLDGaAHHJDfGYnN6STNZdEcqAZIHnr0wjczN5iZo+Z2RNm9pH+qpTIQFPblhzU3YGbWSvwZeB3geOBd5jZ8f1VMZGBorYtuejLCHwe8IS7L3f3vcA1wLn9Uy2RAaW2LVnoSwy8A3i62/1VwItONzCzBcCC2t09P/XrX7x3XtHWc0/WmRc7HHimrpL9R3VoTB3iZbsHp8e2fWC7bp2ytJ49IftbHc9nvItk8+rQ7wZjHZJtu+EfYrr7lcCVAGZ2r7vPbfQ1i6gOqkN/KFu7Lks9VIfm1qEvIZTVwLRu96fW0kRyp7YtWehLB34PMMvMZprZcOCPgZv6p1oiA0ptW7JQdwjF3TvN7M+BW4BW4Cp3f6SHYlfWe71+pDpUqQ6BOtp2WX6PMtRDdahqSh3MXYdXiYjkSEvpRUQypQ5cRCRTTenAy7Is2cyeMrOHzOxBMwuOoO/3a15lZhvM7OFuaePN7FYzW1q7PWwA6vApM1tdey4eNLOzG1yHaWZ2u5k9amaPmNkHa+lNfS76Wxnattr10G3XDe/AS7gs+Ux3P6WJ80S/AbzlgLSPALe5+yzgttr9ZtcB4PLac3GKu9/c4Dp0Ah929+OBM4CLau2g2c9FvylZ21a7ft6QadfNGIEP6WXJ7n4n8OwByecCC2vfLwTmD0Admsrd17r7/bXvtwGLqa54bOpz0c+GbNtWu/5tHQa0XTejA08tS+5ownVTHPiJmd1XWwo9UCa7+9ra9+uAyQNUjz83s0W1t6JNC12Y2QzgVOAuyvNc1KMsbVvt+oWGTLseah9ivsrdT6P6lvciM3vNQFfIq/M4B2Iu578ALwFOAdYC/9CMi5rZKOD7wIfcfWv3vAF8LnKndv28IdWum9GBl2ZZsruvrt1uAH5A9S3wQFhvZlMAarcbml0Bd1/v7l3uXgH+lSY8F2bWRrWRf8fd9x+FMeDPRR+Uom2rXT9vqLXrZnTgpViWbGaHmtno/d8DbwIGage5m4ALat9fANzY7Arsb1w1b6PBz4WZGfB1YLG7f6Fb1oA/F30w4G1b7fqFhly7dveGfwFnA48Dy4CPN+OaiTocDfym9vVIs+oBfI/qW7l9VGOkFwITqH4yvRT4KTB+AOrwLeAhYFGtsU1pcB1eRfVt5CLgwdrX2c1+Lhrwew1o21a7HtrtWkvpRUQyNdQ+xBQRGTTUgYuIZEoduIhIptSBi4hkSh24iEim1IGLiGRKHbiISKb+C1+Mpt02RASZAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "#The returned token IDs are zero-padded. This can easily be turned into a mask\n",
        "plt.subplot(1, 2, 1)\n",
        "plt.pcolormesh(example_tokens)\n",
        "plt.title('Token IDs')\n",
        "\n",
        "plt.subplot(1, 2, 2)\n",
        "plt.pcolormesh(example_tokens != 0)\n",
        "plt.title('Mask')"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "d546fc60",
      "metadata": {
        "id": "d546fc60"
      },
      "source": [
        "### The encoder/decoder model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d6b233f7",
      "metadata": {
        "id": "d6b233f7"
      },
      "outputs": [],
      "source": [
        "embedding_dim = 256\n",
        "units = 1024"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "03baff17",
      "metadata": {
        "id": "03baff17"
      },
      "source": [
        "**The encoder**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "50058341",
      "metadata": {
        "id": "50058341"
      },
      "outputs": [],
      "source": [
        "class Encoder(tf.keras.layers.Layer):\n",
        "  def __init__(self, input_vocab_size, embedding_dim, enc_units):\n",
        "    super(Encoder, self).__init__()\n",
        "    self.enc_units = enc_units\n",
        "    self.input_vocab_size = input_vocab_size\n",
        "\n",
        "    # The embedding layer converts tokens to vectors\n",
        "    self.embedding = tf.keras.layers.Embedding(self.input_vocab_size,\n",
        "                                               embedding_dim)\n",
        "\n",
        "    # The GRU RNN layer processes those vectors sequentially.\n",
        "    self.gru = tf.keras.layers.GRU(self.enc_units,\n",
        "                                   # Return the sequence and state\n",
        "                                   return_sequences=True,\n",
        "                                   return_state=True,\n",
        "                                   recurrent_initializer='glorot_uniform')\n",
        "\n",
        "  def call(self, tokens, state=None):\n",
        "    shape_checker = ShapeChecker()\n",
        "    shape_checker(tokens, ('batch', 's'))\n",
        "\n",
        "    # 2. The embedding layer looks up the embedding for each token.\n",
        "    vectors = self.embedding(tokens)\n",
        "    shape_checker(vectors, ('batch', 's', 'embed_dim'))\n",
        "\n",
        "    # 3. The GRU processes the embedding sequence.\n",
        "    #    output shape: (batch, s, enc_units)\n",
        "    #    state shape: (batch, enc_units)\n",
        "    output, state = self.gru(vectors, initial_state=state)\n",
        "    shape_checker(output, ('batch', 's', 'enc_units'))\n",
        "    shape_checker(state, ('batch', 'enc_units'))\n",
        "\n",
        "    # 4. Returns the new sequence and its state.\n",
        "    return output, state"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "fb369d22",
      "metadata": {
        "id": "fb369d22",
        "outputId": "00e3fbc9-0cf1-44f5-97b8-ab7ee17519a5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Input batch, shape (batch): (64,)\n",
            "Input batch tokens, shape (batch, s): (64, 23)\n",
            "Encoder output, shape (batch, s, units): (64, 23, 1024)\n",
            "Encoder state, shape (batch, units): (64, 1024)\n"
          ]
        }
      ],
      "source": [
        "# Convert the input text to tokens.\n",
        "example_tokens = input_text_processor(example_input_batch)\n",
        "\n",
        "# Encode the input sequence.\n",
        "encoder = Encoder(input_text_processor.vocabulary_size(),\n",
        "                  embedding_dim, units)\n",
        "example_enc_output, example_enc_state = encoder(example_tokens)\n",
        "\n",
        "print(f'Input batch, shape (batch): {example_input_batch.shape}')\n",
        "print(f'Input batch tokens, shape (batch, s): {example_tokens.shape}')\n",
        "print(f'Encoder output, shape (batch, s, units): {example_enc_output.shape}')\n",
        "print(f'Encoder state, shape (batch, units): {example_enc_state.shape}')"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "555a4d0a",
      "metadata": {
        "id": "555a4d0a"
      },
      "source": [
        "**The attention head**"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "941f47c2",
      "metadata": {
        "id": "941f47c2"
      },
      "source": [
        "The decoder uses attention to selectively focus on parts of the input sequence. The attention takes a sequence of vectors as input for each example and returns an \"attention\" vector for each example. This attention layer is similar to a layers.GlobalAveragePoling1D but the attention layer performs a weighted average."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "34d33430",
      "metadata": {
        "id": "34d33430"
      },
      "outputs": [],
      "source": [
        "class BahdanauAttention(tf.keras.layers.Layer):\n",
        "  def __init__(self, units):\n",
        "    super().__init__()\n",
        "    # For Eqn. (4), the  Bahdanau attention\n",
        "    self.W1 = tf.keras.layers.Dense(units, use_bias=False)\n",
        "    self.W2 = tf.keras.layers.Dense(units, use_bias=False)\n",
        "\n",
        "    self.attention = tf.keras.layers.AdditiveAttention()\n",
        "\n",
        "  def call(self, query, value, mask):\n",
        "    shape_checker = ShapeChecker()\n",
        "    shape_checker(query, ('batch', 't', 'query_units'))\n",
        "    shape_checker(value, ('batch', 's', 'value_units'))\n",
        "    shape_checker(mask, ('batch', 's'))\n",
        "\n",
        "    # From Eqn. (4), `W1@ht`.\n",
        "    w1_query = self.W1(query)\n",
        "    shape_checker(w1_query, ('batch', 't', 'attn_units'))\n",
        "\n",
        "    # From Eqn. (4), `W2@hs`.\n",
        "    w2_key = self.W2(value)\n",
        "    shape_checker(w2_key, ('batch', 's', 'attn_units'))\n",
        "\n",
        "    query_mask = tf.ones(tf.shape(query)[:-1], dtype=bool)\n",
        "    value_mask = mask\n",
        "\n",
        "    context_vector, attention_weights = self.attention(\n",
        "        inputs = [w1_query, value, w2_key],\n",
        "        mask=[query_mask, value_mask],\n",
        "        return_attention_scores = True,\n",
        "    )\n",
        "    shape_checker(context_vector, ('batch', 't', 'value_units'))\n",
        "    shape_checker(attention_weights, ('batch', 't', 's'))\n",
        "\n",
        "    return context_vector, attention_weights"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "b4f17d81",
      "metadata": {
        "id": "b4f17d81"
      },
      "source": [
        "**Test the Attention layer**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d8c24b18",
      "metadata": {
        "id": "d8c24b18",
        "outputId": "dad9e60b-fbbc-46d0-fcb4-878eb65f3254"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "TensorShape([64, 23])"
            ]
          },
          "execution_count": 23,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Create a BahdanauAttention layer\n",
        "attention_layer = BahdanauAttention(units)\n",
        "(example_tokens != 0).shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "08325df3",
      "metadata": {
        "id": "08325df3",
        "outputId": "b37a25fc-c6d2-4620-e29f-4d82d132f8bc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Attention result shape: (batch_size, query_seq_length, units):           (64, 2, 1024)\n",
            "Attention weights shape: (batch_size, query_seq_length, value_seq_length): (64, 2, 23)\n"
          ]
        }
      ],
      "source": [
        "# Later, the decoder will generate this attention query\n",
        "example_attention_query = tf.random.normal(shape=[len(example_tokens), 2, 10])\n",
        "\n",
        "# Attend to the encoded tokens\n",
        "\n",
        "context_vector, attention_weights = attention_layer(\n",
        "    query=example_attention_query,\n",
        "    value=example_enc_output,\n",
        "    mask=(example_tokens != 0))\n",
        "\n",
        "print(f'Attention result shape: (batch_size, query_seq_length, units):           {context_vector.shape}')\n",
        "print(f'Attention weights shape: (batch_size, query_seq_length, value_seq_length): {attention_weights.shape}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "981fa577",
      "metadata": {
        "id": "981fa577",
        "outputId": "ebabf7a6-c121-4596-81cd-2a936d50a37d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Mask')"
            ]
          },
          "execution_count": 25,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAeaUlEQVR4nO3df7hcVX3v8ffnnJMfGGJ+ASEGMIARpFWizaXEWn+Atgg+glWxyq1R6Y0+1V7x2qtY673WWovtvYLe+tSbFiRaEShqweqjIkWol1+KIgooAQQh5gcEIkGE5OR87x97R4ZzZu8zs8+efWbN+byeJ8/M7D177TWT71mz5jtr7aWIwMzM0jM03RUwM7Nq3ICbmSXKDbiZWaLcgJuZJcoNuJlZotyAm5klyg14j0n6lKQPTHc92pH0u5J+0uFzXyzpvl7XyQxA0rck/fF016PfDWQDnv/nPyRpzrjtd0t6acvjFZJC0khN532TpG+3bouIt0XEX9VRft0i4j8i4og6ypJ0vqQP11GWpSH/e9olab9x27+f/12tmKaqzRgD14DnQfO7QACvnN7amA28nwKv3/tA0rOBp0xfdWaWgWvAgTcC1wHnA2v3bpT0WeAQ4MuSHpH0HuDqfPeOfNua/LlvkXRb3ov/uqSnt5QTkt4maaOkHZI+qcyzgE8Ba/KyduTPf1LPVNJ/kXSHpAclXSbpaZOVPf4FSpor6Vd7ez6S3i9pVNJT88d/Jemc/P4cSf9L0s8kbc1TOvvk+56UFpH0vLz3tFPSv0i6aHyvWtK7JW2TtFnSm/Nt64DTgPfkr/3L+fb3StqUl/cTScd3/t9oifgs2d/cXmuBz+x9IOmkPKYelnSvpA+27Jsr6Z8lbc/j/TuSlo4/gaRlkm6W9N97+UKSFBED9Q+4A/gT4LeA3cDSln13Ay9tebyCrKc+0rLt5LyMZwEjwF8A17TsD+DfgIVkHwj3Ayfk+94EfHtcfc4HPpzfPw54AHgeMAf4P8DVnZTd5nVeDbw6v/8N4E7g5S37XpXfPxu4DFgMzAe+DPxNvu/FwH35/dnAPcA7gVnAHwC7Wur+YmAU+FC+/0TgUWDR+NeZPz4CuBd4Wst7ffh0x4f/1fq3djfwUuAn+d/LMHAf8PQ8llfkcfNsss7ic4CtwCn58W/N4/Ep+bG/BTw13/ct4I+BQ4HbgXXT/Xr78d9A9cAlvYAseC6OiBvJGrU3dFnM28gauNsiYhT4CLCqtRcOnBUROyLiZ8CVwKoOyz4NOC8ivhcRjwPvI+uxr6hQ9lXAi/L8/XOAT+SP5wL/Cbg6772vA94VEQ9GxM789fxhm/KOJfvA+kRE7I6ILwI3jHvObuBD+f6vAo+QNdTt7CH7kDpK0qyIuDsi7ix6Yyxpe3vhLwNuAzbt3RER34qIH0bEWETcDHweeFG+ezewBHhGROyJiBsj4uGWco8i+xv4nxGxvokXkpqBasDJvr59IyIeyB9fQEsapUNPBz6ef6XbATwICFje8pwtLfcfBfbtsOynkfVyAYiIR4DtFcu+iqx38zzgh8DlZH8YxwJ3RMR2YH+y3s2NLa/na/n2dnXbFHn3J3fvuOdszz/UJq1fRNwBnAF8ENgm6cLWdJENlM+SdZTeREv6BEDSb0u6UtL9kn5B1kHar+W4rwMXSvq5pL+VNKvl8NPIPgwu6fULSNXANOB5XvdUsl7oFklbgHcBR0s6On/a+EsvtrsU473AWyNiYcu/fSLimg6qMdmlHX9O9gGxt87zyHogmwqPKHYNWe/3VcBVEXErWdrlRLLGHbJ0za+A32h5LQsiol2juxlYPi7nfnAX9Znw2iPigojY+60ogI92UZ4lIiLuIfsx80Tgi+N2X0CWwjs4IhaQ/U6k/LjdEfGXEXEU8HzgFTw5n/5Bshi+QNJwT19EogamAQdOIfvafhRZ2mEVWV7uP3giKLYCh7Uccz8wNm7bp4D3SfoNAEkLJL22wzpsBQ6SNLtg/+eBN0tapWyI40eA6yPi7g7L/7WIeBS4EXg7TzTY15D1cK7KnzMG/CNwtqQD8tezXNLvtynyWrL37x2SRiSdDBzTRZWe9N5KOkLScfnrfIzsg2Ssi/IsLacDx0XEL8dtnw88GBGPSTqGlpSmpJdIenbeOD9MllJpjZHdwGuBecBnJA1Se1WLQXpD1gKfjoifRcSWvf+AvwdOy3PFfwP8RZ5O+LO8Efxr4P/l246NiC+R9RQvlPQw8CPg5R3W4d+BW4Atkh4YvzMivgl8APgCWY/3cNrnozt1FdkPije0PJ7PE6NrAN5L9qPsdfnr+SZt8tYRsYvsh8vTgR3Afyb7QfXxDutyLlm+e4ekfyXLf59F1oPaAhxAlvO3ARQRd0bEd9vs+hPgQ5J2Av8DuLhl34Fk6ZGHyXLnV5GlVVrL3RuXS4Hz3Ig/mZ6c8jR7gqTrgU9FxKenuy5mNpE/zezXJL1I0oF5CmUt2eiWr013vcysvVqmkNvAOILsK+484C7gNRGxeXqrZGZFnEIxM0uUUyhmZolqNIUysuApMeeABRN3TLjaR2eKDuuHLxXDd3Q6eMPqspOHHoiIdpOUemq/xcOx4uBZkz9xQNx+s69V1bSi2G60AZ9zwAKO/MRbJmwfVrUWd6jguD1jFT8RarTwpI3TXYUZ55txyT2TP6t+Kw6exQ1fP2Q6Tj0tfv9pR0/+JKtVUWw7hWJmlig34GZmiWo0hbLnVyM8fNN+kz9xiipmZApFWUamYN+Oj1RMxRbVvaQOK/68k8u0mNXj6z//QSPncapmcu6Bm5klyg24mVmiGk2hxJwxxg57tIETVR2X2D5/ESWjWiYueFZeFsBhr7+pi0qZpcNpj2a5B25mlig34GZmiXIDbmaWqI5y4JIWAv8E/CbZQLe3kK1EfRHZytN3A6dGxENl5fzmvg9wzQvPnbB9uOI12odq/vxx/m7mqSu2+5njenB12gJ+HPhaRBwJHE22esaZwBURsRK4In9slhrHtiVr0gZc0gLghWRLZhERuyJiB3AysCF/2gayNSnNkuHYttR1kkI5lGzx30/nq7vfCLwTWNpysf8tZGvWTSBpHbAOYNb8Raz+uz+dcqWfKLxge9WZmGdUPK6dspGMfXC1xDIHnjNjZnZWju3WuD5keX+vi9LUzMl+N4ippE5SKCPA84B/iIjnAr9k3FfKyFaFaNssRcT6iFgdEauH95k31fqa1alybLfG9f5LhhuprNl4nTTg9wH3RcT1+eNLyIJ+q6RlAPnttt5U0axnHNuWtEm/+0XEFkn3SjoiIn4CHA/cmv9bC5yV3146WVkKGN7d5hwl6YYqF6aqu7y6VZ0o2pRt73h+2+0H/P1gpVbqjG3rf2WppFTTK50m7/4U+Jyk2WSL3b6ZrPd+saTTgXuAU3tTRbOecmxbsjpqwCPiJmB1m13H11obs4Y5ti1lnolpZpaoZhd0mD/Gzhe0uRphk0nhsiR4UT1qTpz7aoQ2qFLNJafKPXAzs0S5ATczS1SzU8j2iNGdsydsLs1QlO0ryHhUHUYYQwU7Swqskl25ff0xhfueue6G7gs06xNFQ/WcWukN98DNzBLlBtzMLFFuwM3MEtVoDnzkEbH/tW0u/NMnU8ujcIXiYop6hxg+9JY17XdU+C2gVOlvAe0LrPpaF513baXjbHAM4hUR+yGv7x64mVmi3ICbmSWq2WGEi0bRax6YsHm4aPgeMFZhlmbVrEadmZyFJ22ssTSz/tEPqQPLuAduZpYoN+BmZolqNIUyd2SUZy6+v+fnGWpw0ckta37R2LnMmuI0SRrcAzczS5QbcDOzRLkBNzNLVKM58Me3zuWuc47s/YnKxgOWpMc1VnBIycecXtv9MVUU1a3queouryn7XnTddFdhRhjEmZP9rsrvDn38p2pmZmXcgJuZJarRFErsN8qetdsnbq846m+sIFdSdRhhhWtZFVpwomdi2mDyEMP+4R64mVmi3ICbmSXKDbiZWaI6yoFLuhvYCewBRiNitaTFwEXACuBu4NSIeKisnLnDo6xcNHEq/UjJmLbRmse01T3N3lPp01ZXbA8a57nT0E3r+JKIWBURq/PHZwJXRMRK4Ir8sVmKHNuWpKl0b08GNuT3NwCnTLk2Zv3BsW1J6HQYYQDfkBTA/42I9cDSiNic798CLG13oKR1wDqAkf0WcN1PD23zpOauHliqYPGI0mGOn2u/uWxI4mFv+H7ndbJeqxTbrXF9yPJm10VpQtWZmE69NKvTyHtBRGySdABwuaQft+6MiMj/ACbI/yDWA8w5bHmftNRmv1YptlvjevXRcx3XNi06SqFExKb8dhvwJeAYYKukZQD57bZeVdKsVxzblrJJe+CS5gFDEbEzv/97wIeAy4C1wFn57aWTlbXvnF0ce+hPp1bjFlVGlAyVpGuqrL/pUSjpqjO2B41TIWnoJIWyFPiSsqTuCHBBRHxN0neAiyWdDtwDnNq7apr1hGPbkjZpAx4RdwETPo4jYjtwfC8qZdYEx7alzjMxzcwS1ej4p8dGR/jxgwd0dUxZnrvK1QhHhotnfUZBDrwsNx5faf96Fp7kqxFausqGETo/3j/cAzczS5QbcDOzRDWaQhndPcz2+xZO2F5h9F6p2id2lpRXdK2tbeuPqXSqZ667odJxZk2pMkvTaZfecA/czCxRbsDNzBLlBtzMLFGN5sDn7fM4a549cXhd2fT2ssUedo0Nt90+d3h34TGjBcdU5an0Noics06De+BmZolyA25mlqhGUyiP7pzLd698Vu9PVDYssWyIYYXhjPpwwWlKyiob5lhlSOWK91/T/UFmJaou6FAnp3Em5x64mVmi3ICbmSWq0RRKjAS79xvt7pgmZ2kW7SupQwwVHFSx4p6JaYPI6ZDecA/czCxRbsDNzBLlBtzMLFGN5sAZCobnt5klWfe4urrVfHnDw15/U63lmfUL57qb5R64mVmi3ICbmSWq2RTK7iFiy9yJ28uyJHWnV8rKGysor+Yszp1nr+n6mMPfdW29lTDrAS/20Cz3wM3MEuUG3MwsUW7AzcwS1XEOXNIw8F1gU0S8QtKhwIXAEuBG4I8iYldZGcO7YN+fTUwolw/SK7usX8Hm0rR5hYR23YskV7DljOc3dq4Dz5k5VzesI65tapq68uEg5tq76YG/E7it5fFHgbMj4hnAQ8DpdVbMrCGOa0tWRw24pIOAk4B/yh8LOA64JH/KBuCUHtTPrGcc15a6TlMo5wDvAebnj5cAOyJi76UF7wOWtztQ0jpgHcDseYuY+2Af5COs1C/e2P0wxyoWfGbah0aeQw1xfcjyZkfjWjVNLlLRVLpm0h64pFcA2yLixioniIj1EbE6IlaPzJ1XpQiz2tUZ1/svqXehbLNOddJ1+B3glZJOBOYCTwU+DiyUNJL3Vg4CNvWumma1c1xb8iZtwCPifcD7ACS9GPiziDhN0r8AryH7xX4tcOlkZY3uAw+sarOj6kzHCgsw1F5e0ZCXmi/C5ZmY9aozrm1qBnF0SFOmMg78vcB/k3QHWe7w3HqqZDatHNeWjK5+fYmIbwHfyu/fBRxTf5XMmuW4tlR5JqaZWaKaHf80HIzN29PVIWWp5ML0c9FCw5MUWPO6DZWsfKsXNbbB4zx3b7gHbmaWKDfgZmaJ6vspZBqrckxxmqQsJVO0r9+X7DSzmck9cDOzRLkBNzNLVLMplKFgeP7uidurDv+oO3/R0KzKw97w/VrLM+sXHm3SLPfAzcwS5QbczCxRbsDNzBLVbA58TOzZOau24gqH/ZUNPawwjLDucYS3ry++1MYz13kmpqWraNEE58Z7wz1wM7NEuQE3M0tUoymUWTvFgVfVt/xUlZmTTak68nDnG5pZj7JuVS46Vmb+BV7AYpA0uR5lP6s7leQeuJlZotyAm5klyg24mVmiGs2Bx+JRdp+2fcL2ocLVhKtRST52qCQhO1aQyI0K1Vt40sbuDzJLgIcE9g/3wM3MEuUG3MwsUY2mUMYeGeGxq/abuKPqxf6KUhtl5TU0xPDR9+5f7cCG6ve0v72mmRPZwOn3IYEzKcXjHriZWaLcgJuZJarRFMoBS3bwX9/8rxO2z9KewmOGS65MNUT7fWMln0u7o/uZoBcdeWDXx5ilYCalGwbRpD1wSXMl3SDpB5JukfSX+fZDJV0v6Q5JF0ma3fvqmtXHsW2p6ySF8jhwXEQcDawCTpB0LPBR4OyIeAbwEHB6z2pp1huObUvapA14ZB7JH87K/wVwHHBJvn0DcEovKmjWK45tS11HOXBJw8CNwDOATwJ3AjsiYjR/yn3A8oJj1wHrAGY/ZRHnf/CVU63zlJQt9hBFH2ev60lVeq/u4Zl9YN+Lrqu1vKqx3RrXhyxvdl2UOvX7kMCZpMrvER2NQomIPRGxCjgIOAY4stMTRMT6iFgdEatnzZnXdQXNeqlqbLfG9f5L6rtEslk3uhpGGBE7gCuBNcBCSXu7HgcBm+qtmllzHNuWokm/+0naH9gdETsk7QO8jOxHniuB1wAXAmuBSycrK5aMMvrGiRezqqroolVlF58qu9BVnRac6ItZ9bs6Y3sm8dDD/tFJ8m4ZsCHPFQ4BF0fEv0m6FbhQ0oeB7wPn9rCeZr3g2LakTdqAR8TNwHPbbL+LLGdoliTHtqXOU+nNzBLV6PinOSOjPGPRAxO2zx4qnko/Wji2r3ghiLGS8XN1Lx6xZc0vai3PrB84z50G98DNzBLlBtzMLFGNplAeHx3h9ocmLnQwXLJOZZUhgXUPIyxdE/MrB7Td7DUxLWVlMzSdXukf7oGbmSXKDbiZWaKaXRPz4RF2Xd5mTcwyVWZO1nwxpipLbG45o3hNzD5YsrPUged4vUwrluoFsAYx9eMeuJlZotyAm5klyg24mVmiGs2BP2XxYzzndbd2dcxYlMyqLBl+WGSkZEWHonOVnefnxz7cdR3M+t0g5osHkXvgZmaJcgNuZpaoRlMoj42O8OMH289cLFLl4lNlsy3LZlUODxVcHKskjcNXl7bd7AUdLGWeiZkG98DNzBLlBtzMLFFuwM3MEtX4gg7PXHT/hO1lw/TKcuBVhhGW5bPLFoIosm3Njq6PMet3znOnwT1wM7NEuQE3M0tUoymUXz46h2tvXjlhe8myl5RmSQr2lZVXquhkZcMI13dX1GRWvvWGagea1ajuKw46JdMb7oGbmSXKDbiZWaIaTaEMPSbm/7jGU/bBYg9123LG89tub/KlekEHq1s/LAIxiGmcSXvgkg6WdKWkWyXdIumd+fbFki6XtDG/XdT76prVx7FtqeskhTIKvDsijgKOBd4u6SjgTOCKiFgJXJE/NkuJY9uSNmkDHhGbI+J7+f2dwG3AcuBkYEP+tA3AKT2qo1lPOLYtdV0lpCWtAJ4LXA8sjYjN+a4tQNvL8klaB6wDGF68kJ1HjrZ5UvE5y0bw1a1orYcqwxI9jDAt3cZ2a1wfsrzRn5KSNIj5537QcdMkaV/gC8AZEfGkZWgiIij4zSwi1kfE6ohYPbzvvClV1qwXqsR2a1zvv2S4oZqaPVlHDbikWWQB/rmI+GK+eaukZfn+ZcC23lTRrHcc25aySb/7SRJwLnBbRHysZddlwFrgrPz20knL2iNmbW9zygbTJGUKl8ssS4cU1L1q6ufuv24/jLDMivd72F8Vdca2lasyjNBpl8l1krz7HeCPgB9Kuinf9udkwX2xpNOBe4BTe1JDs95xbFvSJm3AI+LbFPeRj6+3OmbNcWxb6jyV3swsUY2Ofzpw4UO879VfmLB9dxT/ij9cmJiutuBxmaJz/fMRy2s9j1m/cJ45be6Bm5klyg24mVmiGk2h3P/4fD5554smbB8umbYYJVmS4aH2O8vWvSyzp+C4oa90n6pZeNLGSnUwa5KH96XNPXAzs0S5ATczS1SjKZS5I6McuXjirOS6R5OUGasw7bOsflvW/GIq1THrS06TpME9cDOzRLkBNzNLlBtwM7NENZoDf+SXc7n2umd1d1DZyghVhgtWWWmh7DwfK95VXIfuDzn8XddWOJFZNU0uQux8e3XugZuZJcoNuJlZohpNoWgUZj/YZf5Ada/2UKG8Cgs6VK5CwbW77v1AyUIPBeUd/CEv9GD9z7NBq3MP3MwsUW7AzcwS5QbczCxRjebAh+eNMv/Y+ydsrz3NXVHh1QhLkuBFdV9woq9GaIPJ+ef+4R64mVmi3ICbmSWq0RRK7Bhhz6X7T9xRdwqlbNhfhSGBeyrUb/u6Nq9zkvOUqnsoY80XgFyy3kMWZ4omZ2n2g35OGbkHbmaWKDfgZmaJanZBh/0e44g33TZh++yhPYXHlK1vOVJw3FgUfy7tGhsuqWF7QyUXwNq2ZkfX5Zn1u35OG9gTJu2BSzpP0jZJP2rZtljS5ZI25reLeltNs/o5ti11naRQzgdOGLftTOCKiFgJXJE/NkvN+Ti2LWGTNuARcTXw4LjNJwMb8vsbgFPqrZZZ7zm2LXVVc+BLI2Jzfn8LsLToiZLWAesAZs1fxI8u7m5Bh9pHGJYVWGVo3RkVjmly2GQFB54zo4cEdhTbrXF9yPJGf0pqxCAOFRzEvP6UR6FERFDShETE+ohYHRGrh/eZN9XTmTWmLLZb43r/Jd3/MG5Wh6oN+FZJywDy2231VclsWjm2LRlVv/tdBqwFzspvL+3koLF9gp3PGp2wvSytUfeSmHWrssRmmZVvvaHeAq1blWLbyg1i+qIfdDKM8PPAtcARku6TdDpZcL9M0kbgpfljs6Q4ti11k/bAI+L1BbuOr7kuZo1ybFvqPJXezCxRjY5/mvWwWPbvbT4z6h7aV5EKFhQuUzJrv5JHXnds+x198h5VUlT3CvXe96LrplQVmx6DOCyxblV+J3AP3MwsUW7AzcwS1WgKZXQf2P7s7j4zqgwxrDq8sLC8mj/mKg2bLEk3rHj/jJ45aQPKQw8n5x64mVmi3ICbmSWq2TUx54wxeuivJmxXSUohSlIHZccVH9T90AeVHBNF+ZCSPMlhb/h+13UwS4HTHs1yD9zMLFFuwM3MEuUG3MwsUc3OxNwxxLIvzunqmKpXKqyi6Fx1n+fRVxfMtoTipH+lhH9F/VCHKi65ZLprMON5xmVvDC9rv909cDOzRLkBNzNLVKMplN0Lx9j8B49P2F72zVxDxVeYKhzCV1VDK0R4GKENKg8j7JWNbbe6B25mlig34GZmiXIDbmaWqEZz4Owegq1zJ2wuSz03ulZB0clqnrJ/59lrui7u8HddW6ESZs2qMozQefPq3AM3M0uUG3Azs0Q1m0IR7dMRTU2pnOxchVcWrLkOZfp8sqOZ9Q/3wM3MEuUG3MwsUY2mUA586g7ee8KlE7bP0p5K5Y0V5BvmaHfhMbuj+5f8+SMLriRjljiPAEnblHrgkk6Q9BNJd0g6s65KmU03x7aloHIDLmkY+CTwcuAo4PWSjqqrYmbTxbFtqZhKD/wY4I6IuCsidgEXAifXUy2zaeXYtiRMJQe+HLi35fF9wG+Pf5KkdcC6/OHjbzvy6h9N4Zx12A94wHUYyDo8vaZyJo3t8XE9vGzjdMc1VHo/21/lrtk61G4Q69A2tnv+I2ZErAfWA0j6bkSs7vU5y7gOrkMd+i2u+6UerkOzdZhKCmUTcHDL44PybWapc2xbEqbSgH8HWCnpUEmzgT8ELqunWmbTyrFtSaicQomIUUnvAL4ODAPnRcQtkxy2vur5auQ6ZFyHAhViu19eRz/Uw3XINFIHRdEK5GZm1tc8ld7MLFFuwM3MEtVIA94v05Il3S3ph5JukvTdhs55nqRtkn7Usm2xpMslbcxvF01DHT4oaVP+Xtwk6cQe1+FgSVdKulXSLZLemW9v9L2oWz/EtuN65sZ1zxvwPpyW/JKIWNXgONHzgRPGbTsTuCIiVgJX5I+brgPA2fl7sSoivtrjOowC746Io4BjgbfncdD0e1GbPottx/UTZkxcN9EDn9HTkiPiauDBcZtPBjbk9zcAp0xDHRoVEZsj4nv5/Z3AbWQzHht9L2o2Y2Pbcf3rOkxrXDfRgLeblry8gfO2E8A3JN2YT4WeLksjYnN+fwuwdJrq8Q5JN+dfRRtLXUhaATwXuJ7+eS+q6JfYdlw/2YyJ65n2I+YLIuJ5ZF953y7phdNdocjGcU7HWM5/AA4HVgGbgf/dxEkl7Qt8ATgjIh5u3TeN70XqHNdPmFFx3UQD3jfTkiNiU367DfgS2Vfg6bBV0jKA/HZb0xWIiK0RsScixoB/pIH3QtIssiD/XER8Md887e/FFPRFbDuunzDT4rqJBrwvpiVLmidp/t77wO8B03UFucuAtfn9tcDEZYp6bG9w5V5Fj98LSQLOBW6LiI+17Jr292IKpj22HddPNuPiOiJ6/g84EbgduBN4fxPnbFOHw4Af5P9uaaoewOfJvsrtJsuRng4sIftleiPwTWDxNNThs8APgZvzYFvW4zq8gOxr5M3ATfm/E5t+L3rwuqY1th3XMzuuPZXezCxRM+1HTDOzgeEG3MwsUW7AzcwS5QbczCxRbsDNzBLlBtzMLFFuwM3MEvX/AcN2aDrvJyozAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "plt.subplot(1, 2, 1)\n",
        "plt.pcolormesh(attention_weights[:, 0, :])\n",
        "plt.title('Attention weights')\n",
        "\n",
        "plt.subplot(1, 2, 2)\n",
        "plt.pcolormesh(example_tokens != 0)\n",
        "plt.title('Mask')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d8e85ab8",
      "metadata": {
        "id": "d8e85ab8"
      },
      "outputs": [],
      "source": [
        "attention_weights.shape\n",
        "attention_slice = attention_weights[0, 0].numpy()\n",
        "attention_slice = attention_slice[attention_slice != 0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ec43c7ef",
      "metadata": {
        "id": "ec43c7ef",
        "outputId": "73dbf4e5-d132-4d80-bb66-0ea5fa445b20"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x275e5ff0ee0>]"
            ]
          },
          "execution_count": 27,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 0 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAFzCAYAAADMjJRjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAk9klEQVR4nO3dfbRdVX3u8e8DMWhBwELKkJcaqmgbX4pyQKlCVa4WagXbwhXkVtLSUkvpy7W0xnYMRLQttOJLC1VTUfCFIkXtTUsqekXFVqUJFIMBsTEGOGgHEVHLtQqR3/1jr4ObyUnOTs4+2fsk388Ye2StueZa+7eTw+Q5a8+1VqoKSZIkST+0y6gLkCRJksaNIVmSJElqGJIlSZKkhiFZkiRJahiSJUmSpIYhWZIkSWosGHUBrX333bcWL1486jIkaZvccMMN36iqRaOuY3ty3JY0X21pzB67kLx48WJWr1496jIkaZskuX3UNWxvjtuS5qstjdlOt5AkSZIahmRJkiSpYUiWJEmSGoZkSZIkqWFIliRJkhqGZEmSJKlhSJYkSZIahmRJkiSpYUiWJEmSGoZkSZIkqWFIliRJkhqGZEmSJKlhSJYkSZIaC0ZdgLSjWbzs6lGXwIbzXzLqEiRJmtc8kyxJkiQ1PJO8HXmGUeNi1D+L/hxKksadIVkPY3jSOBj1zyH4syhJOztDsuYVw5MkSdoedpiQbHiSJEnSsAx04V6SY5PclmRdkmXTbD86yY1JNiU5cZrteyaZTHLRMIqWJEmS5tKMITnJrsDFwHHAEuCUJEuabncAS4HLN3OYNwDXbXuZkiRJ0vYzyJnkI4B1VbW+qu4HrgBO6O9QVRuqag3wYLtzksOA/YCPDaFeSZIkac4NEpIPAO7sW5/s2maUZBfgQuDsGfqdkWR1ktUbN24c5NCStNObzVS4JD9IclP3WtHXfnCS67tjfjDJwu3xWSRp3Mz1w0TOBFZW1eSWOlXV8qqaqKqJRYsWzXFJkjT/DWEq3H9X1aHd6/i+9guAt1TVk4B7gdOHXrwkzQOD3N3iLuCgvvUDu7ZBHAkcleRMYA9gYZL7quoRZzwkSVvloalwAEmmpsLdMtWhqjZ02x4xFW46SQK8EHhF13QZcC7w9mEVLUnzxSBnklcBh3RfwS0ETgZWzLAPAFV1alX9eFUtpjfl4r0GZEkaim2eCtd5dDfN7fNJXta17QN8q6o2zXRMp8lJ2tHNGJK7wfIs4BrgVuDKqlqb5LwkxwMkOTzJJHAS8M4ka+eyaEnSrD2hqibonTV+a5Inbs3OTpOTtKMb6GEiVbUSWNm0ndO3vIreNIwtHeNS4NKtrlCSNJ3ZTIWjqu7q/lyf5FPAM4EPAXsnWdCdINmqY0rSjmSuL9yTJM2NbZ4Kl+RxSXbrlvcFngvcUlUFfBKYuhPGacD/GXrlkjQPGJIlaR6a5VS4nwJWJ/kCvVB8flVNXfD3GuDVSdbRm6N8yfb7VJI0PgaabiFJGj/bOhWuqj4LPH0zx1xP784ZkrRT80yyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktRIVY26hod57GMfW4cddthW7/f59ffMQTVb5zk/sc8Wt1vjzMa9PrDGYRj3+mDmGjfn05/+9A1VNTHkcsbaxMRErV69etRlSNJWS7LZMdszyZIkSVJjwagLaD3lKU/hU5/61Fbvt3jZ1cMvZit96vyXbHG7Nc5s3OsDaxyGca8PZq5xc5IMuRJJ0ih4JlmSJElqGJIlSZKkhiFZkiRJahiSJUmSpIYhWZIkSWoYkiVJkqSGIVmSJElqGJIlSZKkhiFZkiRJahiSJUmSpIYhWZIkSWoYkiVJkqSGIVmSJElqDBSSkxyb5LYk65Ism2b70UluTLIpyYl97Ycm+VyStUnWJHn5MIuXJEmS5sKMITnJrsDFwHHAEuCUJEuabncAS4HLm/bvAq+sqqcCxwJvTbL3LGuWJEmS5tSCAfocAayrqvUASa4ATgBumepQVRu6bQ/271hVX+5b/lqSu4FFwLdmW7gkSZI0VwaZbnEAcGff+mTXtlWSHAEsBL4yzbYzkqxOsnrjxo1be2hJ2ilt61S4vu17JplMclFf2ylJbu6myH00yb5z/TkkaRxtlwv3kjweeB/wq1X1YLu9qpZX1URVTSxatGh7lCRJ89osp8JNeQNwXd8xFwBvA15QVc8A1gBnDbdySZofBgnJdwEH9a0f2LUNJMmewNXAn1TV57euPEnSZjw0Fa6q7gempsI9pKo2VNUa4BEnJ5IcBuwHfKy/uXvtniTAnsDX5qh+SRprg4TkVcAhSQ5OshA4GVgxyMG7/h8B3ltVV217mZKkxjZPhUuyC3AhcHZ/e1U9APwWcDO9cLwEuGQYxUrSfDNjSK6qTfS+brsGuBW4sqrWJjkvyfEASQ5PMgmcBLwzydpu9/8JHA0sTXJT9zp0Lj6IJGlgZwIrq2qyvzHJo+iF5GcC+9ObbvHa6Q7gtSSSdnSD3N2CqloJrGzazulbXkVvGka73/uB98+yRknSI81mKtyRwFFJzgT2ABYmuQ/4EEBVfQUgyZXAIy4I7PosB5YDTExM1LZ8AEkaZwOFZEnS2HloKhy9cHwy8IpBdqyqU6eWkywFJqpqWZL9gSVJFlXVRuBF9L5BlKSdjo+llqR5aJZT4TZ3zK8BrweuS7IGOBT4szn8GJI0tjyTLEnz1LZOhWv6Xwpc2rf+DuAdw6xTkuYjzyRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJjYFCcpJjk9yWZF2SZdNsPzrJjUk2JTmx2XZakv/oXqcNq3BJkiRprswYkpPsClwMHAcsAU5JsqTpdgewFLi82fdHgdcBzwaOAF6X5HGzL1uSJEmaO4OcST4CWFdV66vqfuAK4IT+DlW1oarWAA82+/4c8PGq+mZV3Qt8HDh2CHVLkiRJc2aQkHwAcGff+mTXNoiB9k1yRpLVSVZv3LhxwENL0s5tNlPhuu17JplMclFf28Iky5N8OcmXkvzyXH8OSRpHY3HhXlUtr6qJqppYtGjRqMuRpLE3m6lwfd4AXNe0/Qlwd1U9uTvup4dVsyTNJ4OE5LuAg/rWD+zaBjGbfSVJmzebqXAkOQzYD/hYs+nXgD/v9n+wqr4xF8VL0rgbJCSvAg5JcnCShcDJwIoBj38N8OIkj+su2Htx1yZJmp1tngqXZBfgQuDspn3vbvEN3TSNv0+y3xBqlaR5Z8aQXFWbgLPohdtbgSuram2S85IcD5Dk8CSTwEnAO5Os7fb9Jr2v81Z1r/O6NknS6JwJrKyqyaZ9Ab1v/D5bVc8CPge8aboDeC2JpB3dgkE6VdVKYGXTdk7f8ip6A+t0+74bePcsapQkPdJsprMdCRyV5ExgD2BhkvuA1wLfBT7c9ft74PTpDlBVy4HlABMTE7XV1UvSmBsoJEuSxs5DU+HoheOTgVcMsmNVnTq1nGQpMFFVy7r1fwSeD1wLHAPcMtSqJWmeGIu7W0iSts5spsLN4DXAuUnWAL8C/MHcfAJJGm+eSZakeWo2U+H6+lwKXNq3fjtw9DDrlKT5yDPJkiRJUsOQLEmSJDUMyZIkSVLDkCxJkiQ1DMmSJElSw5AsSZIkNQzJkiRJUsOQLEmSJDUMyZIkSVLDkCxJkiQ1DMmSJElSw5AsSZIkNQzJkiRJUsOQLEmSJDUMyZIkSVLDkCxJkiQ1DMmSJElSw5AsSZIkNQzJkiRJUsOQLEmSJDUMyZIkSVLDkCxJkiQ1DMmSJElSw5AsSZIkNQzJkiRJUsOQLEmSJDUMyZIkSVLDkCxJkiQ1DMmSJElSw5AsSZIkNQzJkiRJUmOgkJzk2CS3JVmXZNk023dL8sFu+/VJFnftj0pyWZKbk9ya5LVDrl+SJEkauhlDcpJdgYuB44AlwClJljTdTgfuraonAW8BLujaTwJ2q6qnA4cBvzkVoCVJkqRxNciZ5COAdVW1vqruB64ATmj6nABc1i1fBRyTJEABuydZADwGuB/4zlAqlyRJkubIICH5AODOvvXJrm3aPlW1Cfg2sA+9wPz/gK8DdwBvqqpvtm+Q5Iwkq5Os3rhx41Z/CEnaGQ0wFe7oJDcm2ZTkxGm275lkMslF02xbkeSLc1W7JI27ub5w7wjgB8D+wMHAHyT5ibZTVS2vqomqmli0aNEclyRJ89+AU+HuAJYCl2/mMG8Arpvm2L8E3De0YiVpHlowQJ+7gIP61g/s2qbrM9lNrdgLuAd4BfDRqnoAuDvJvwITwPrZFi5JO7mHpsIBJJmaCnfLVIeq2tBte7DdOclhwH7AR+mNy1PtewCvBs4Arpy78qXxsHjZ1aMugQ3nv2TUJWgag5xJXgUckuTgJAuBk4EVTZ8VwGnd8onAtVVV9M5ivBAgye7Ac4AvDaNwSdrJDTIVblpJdgEuBM6eZvMbum3fnW2BkjSfzRiSuznGZwHXALcCV1bV2iTnJTm+63YJsE+SdfTOQEzNjbsY2CPJWnph+z1VtWbYH0KStFXOBFZW1WR/Y5JDgSdW1UdmOoDXkkja0Q0y3YKqWgmsbNrO6Vv+Hr3bvbX73TdduyRp1gaZCrc5RwJHJTkT2ANYmOQ+4HZgIskGev9/+LEkn6qq57cHqKrlwHKAiYmJ2tYPIUnjaqCQLEkaOw9NhaMXjk+mdx3IjKrq1KnlJEuBiaqa+gbw7V37YuCfpgvIkrQz8LHUkjQPDTIVLsnhSSbpfaP3zm7qmyRpAJ5JlqR5aoCpcKvoTcPY0jEuBS6dpn0D8LQhlClJ85JnkiVJkqSGIVmSJElqGJIlSZKkhiFZkiRJahiSJUmSpIYhWZIkSWoYkiVJkqSGIVmSJElqGJIlSZKkhiFZkiRJahiSJUmSpIYhWZIkSWoYkiVJkqTGglEXIEmSpM1bvOzqUZfAhvNfMuoStjtDsiRJ0zCYSDs3p1tIkiRJDUOyJEmS1DAkS5IkSQ3nJEuSJGlWRj2Hfy7m73smWZIkSWoYkiVJkqSGIVmSJElqGJIlSZKkhhfuSZI0T+2IF0tJ48IzyZIkSVLDkCxJkiQ1DMmSJElSw5AsSZIkNQzJkiRJUsOQLEmSJDUMyZIkSVJjoPskJzkWeBuwK/Cuqjq/2b4b8F7gMOAe4OVVtaHb9gzgncCewIPA4VX1vWF9AEmSNL68l7PmqxnPJCfZFbgYOA5YApySZEnT7XTg3qp6EvAW4IJu3wXA+4FXVdVTgecDDwytekmSJGkODDLd4ghgXVWtr6r7gSuAE5o+JwCXdctXAcckCfBiYE1VfQGgqu6pqh8Mp3RJkiRpbgwSkg8A7uxbn+zapu1TVZuAbwP7AE8GKsk1SW5M8kfTvUGSM5KsTrJ648aNW/sZJEmSpKGa6wv3FgDPA07t/vzFJMe0napqeVVNVNXEokWL5rgkSdoxJDk2yW1J1iVZNs32o7sTFJuSnDjN9j2TTCa5qFv/kSRXJ/lSkrVJzm/3kaSdxSAh+S7goL71A7u2aft085D3oncB3yRwXVV9o6q+C6wEnjXboiVpZzfg9SJ3AEuByzdzmDcA1zVtb6qqnwSeCTw3yXFDK1qS5pFBQvIq4JAkBydZCJwMrGj6rABO65ZPBK6tqgKuAZ7enZ1YAPwscMtwSpekndqM14tU1YaqWkPvzkIPk+QwYD/gY339v1tVn+yW7wdupHdiRJJ2OjOG5G6O8Vn0Au+twJVVtTbJeUmO77pdAuyTZB3wamBZt++9wJvpBe2bgBurarT3gpGkHcMg14tMK8kuwIXA2VvoszfwUuATm9nutSSSdmgD3Se5qlbSmyrR33ZO3/L3gJM2s+/76d0GTpI0Hs4EVlbVZO9GRA/XffP3d8BfVdX66Q5QVcuB5QATExM1h7VK0kgMFJIlSWNnkOtFNudI4KgkZwJ7AAuT3FdVUxf/LQf+o6reOqxiJWm+MSRL0vz00PUi9MLxycArBtmxqk6dWk6yFJiYCshJ3kjv4utfH3bBkjSfzPUt4CRJc2CQ60WSHJ5kkt50uHcmWbulYyY5EPgTenfLuDHJTUkMy5J2Sp5JlqR5aoDrRVYxw90pqupS4NJueRJ45CRlSdoJeSZZkiRJangmWZI0EouXjfaOoBvOf8lI31/SePNMsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJjYFCcpJjk9yWZF2SZdNs3y3JB7vt1ydZ3Gz/8ST3JTl7SHVLkiRJc2bGkJxkV+Bi4DhgCXBKkiVNt9OBe6vqScBbgAua7W8G/nn25UqSJElzb5AzyUcA66pqfVXdD1wBnND0OQG4rFu+CjgmSQCSvAz4KrB2KBVLkiRJc2yQkHwAcGff+mTXNm2fqtoEfBvYJ8kewGuA18++VEmSJGn7mOsL984F3lJV922pU5IzkqxOsnrjxo1zXJIk7RgGuF7k6CQ3JtmU5MRptu+ZZDLJRX1thyW5uTvmX019KyhJO5tBQvJdwEF96wd2bdP2SbIA2Au4B3g28BdJNgC/D/xxkrPaN6iq5VU1UVUTixYt2trPIEk7nQGvF7kDWApcvpnDvAG4rml7O/AbwCHd69ghlSxJ88ogIXkVcEiSg5MsBE4GVjR9VgCndcsnAtdWz1FVtbiqFgNvBf6sqi5CkjRbM14vUlUbqmoN8GC7c5LDgP2Aj/W1PR7Ys6o+X1UFvBd42dx9BEkaXzOG5G6O8VnANcCtwJVVtTbJeUmO77pdQm8O8jrg1cAjvvaTJA3VINeLTCvJLsCFQHtbzgO648x4TKfJSdrRLRikU1WtBFY2bef0LX8POGmGY5y7DfVJkobvTGBlVU1u65TjqloOLAeYmJioIdYmSWNhoJAsSRo7g1wvsjlHAkclORPYA1iY5D7gbd1xtuWYkrRDMSRL0vz00PUi9ILsycArBtmxqk6dWk6yFJioqmXd+neSPAe4Hngl8NdDrluS5oW5vgWcJGkODHK9SJLDk0zSmw73ziSDPNTpTOBdwDrgK/i0VEk7Kc8kS9I8NcD1Iqt4+PSJ6Y5xKXBp3/pq4GnDrFOS5iPPJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSQ1DsiRJktQYKCQnOTbJbUnWJVk2zfbdknyw2359ksVd+4uS3JDk5u7PFw65fkmSJGnoZgzJSXYFLgaOA5YApyRZ0nQ7Hbi3qp4EvAW4oGv/BvDSqno6cBrwvmEVLkmSJM2VQc4kHwGsq6r1VXU/cAVwQtPnBOCybvkq4Jgkqap/r6qvde1rgcck2W0YhUuSJElzZZCQfABwZ9/6ZNc2bZ+q2gR8G9in6fPLwI1V9f1tK1WSJEnaPrbLhXtJnkpvCsZvbmb7GUlWJ1m9cePG7VGSJM17A1wvcnSSG5NsSnJiX/sTuvabkqxN8qq+bad015GsSfLRJPtur88jSeNkkJB8F3BQ3/qBXdu0fZIsAPYC7unWDwQ+Aryyqr4y3RtU1fKqmqiqiUWLFm3dJ5CkndCA14vcASwFLm/avw4cWVWHAs8GliXZvxu/3wa8oKqeAawBzpqzDyFJY2yQkLwKOCTJwUkWAicDK5o+K+hdmAdwInBtVVWSvYGrgWVV9a9DqlmSNMD1IlW1oarWAA827ff3TX3bjR/+vyDda/ckAfYEvoYk7YRmDMndHOOzgGuAW4Erq2ptkvOSHN91uwTYJ8k64NXA1Nd+ZwFPAs7pvta7KcmPDf1TSNLOZ5DrRTYryUFJ1nTHuKCqvlZVDwC/BdxMLxwvoTe+T7e/0+Qk7dAWDNKpqlYCK5u2c/qWvwecNM1+bwTeOMsaJUlDVlV3As9Isj/wD0muAr5JLyQ/E1gP/DXwWqYZx6tqObAcYGJiorZX3ZK0vfjEPUmanwa5XmRG3W06vwgcBRzatX2lqgq4EviZWVcqSfOQIVmS5qdBrheZVpIDkzymW34c8DzgNnohe0mSqSuoX0Rvmp0k7XQGmm4hSRovVbUpydT1IrsC7566XgRYXVUrkhxO7+5CjwNemuT1VfVU4KeAC5MUvQv13lRVNwMkeT1wXZIHgNvp3R1DknY6hmRJmqcGuF5kFb1pGO1+HweesZljvgN4x3ArlaT5x+kWkiRJUsOQLEmSJDUMyZIkSVLDkCxJkiQ1DMmSJElSw5AsSZIkNQzJkiRJUsOQLEmSJDUMyZIkSVLDkCxJkiQ1DMmSJElSw5AsSZIkNQzJkiRJUsOQLEmSJDUMyZIkSVLDkCxJkiQ1DMmSJElSw5AsSZIkNQzJkiRJUsOQLEmSJDUMyZIkSVLDkCxJkiQ1DMmSJElSw5AsSZIkNQzJkiRJUsOQLEmSJDUMyZIkSVLDkCxJkiQ1DMmSJElSw5AsSZIkNQYKyUmOTXJbknVJlk2zfbckH+y2X59kcd+213bttyX5uSHWLkmSJM2JGUNykl2Bi4HjgCXAKUmWNN1OB+6tqicBbwEu6PZdApwMPBU4Fvib7niSJEnS2BrkTPIRwLqqWl9V9wNXACc0fU4ALuuWrwKOSZKu/Yqq+n5VfRVY1x1PkiRJGluDhOQDgDv71ie7tmn7VNUm4NvAPgPuK0mSJI2VVNWWOyQnAsdW1a93678CPLuqzurr88Wuz2S3/hXg2cC5wOer6v1d+yXAP1fVVc17nAGc0a0+Bbht9h9tm+wLfGNE7z2Ica8PrHEYxr0+sMYteUJVLRrB+45Mko3A7SN463H/ORz3+sAah2Xcaxz3+mAMx+wFA+x8F3BQ3/qBXdt0fSaTLAD2Au4ZcF+qajmwfIBa5lSS1VU1Meo6Nmfc6wNrHIZxrw+sUQ83ql8Kxv3feNzrA2sclnGvcdzrg/GscZDpFquAQ5IcnGQhvQvxVjR9VgCndcsnAtdW7xT1CuDk7u4XBwOHAP82nNIlSZKkuTHjmeSq2pTkLOAaYFfg3VW1Nsl5wOqqWgFcArwvyTrgm/SCNF2/K4FbgE3Ab1fVD+bos0iSJElDMch0C6pqJbCyaTunb/l7wEmb2fdPgT+dRY3b08infMxg3OsDaxyGca8PrFHjYdz/jce9PrDGYRn3Gse9PhjDGme8cE+SJEna2fhYakmSJKlhSGbmx26PWpJ3J7m7u9XeWEpyUJJPJrklydokvzfqmvoleXSSf0vyha6+14+6ps1JsmuSf0/yT6OuZTpJNiS5OclNSVaPup5Wkr2TXJXkS0luTXLkqGvScDlmz55j9vA4Zs/euI7bO/10i+4x2V8GXkTvYSergFOq6paRFtYnydHAfcB7q+ppo65nOkkeDzy+qm5M8ljgBuBl4/L32D0Bcvequi/Jo4B/AX6vqj4/4tIeIcmrgQlgz6r6hVHX00qyAZioqrG852aSy4DPVNW7ujvy/EhVfWvEZWlIHLOHwzF7eByzZ29cx23PJA/22O2Rqqrr6N01ZGxV1der6sZu+b+AWxmjpytWz33d6qO619j9hpjkQOAlwLtGXct8lGQv4Gh6d9yhqu4fh4FWQ+WYPQSO2cPhmD174zxuG5J9dPbQJVkMPBO4fsSlPEz3ldhNwN3Ax6tqrOrrvBX4I+DBEdexJQV8LMkN3dMyx8nBwEbgPd3Xn+9Ksvuoi9JQOWYPmWP2rLwVx+zZGttx25CsoUqyB/Ah4Per6jujrqdfVf2gqg6l9+THI5KM1degSX4BuLuqbhh1LTN4XlU9CzgO+O3uq+VxsQB4FvD2qnom8P+AsZuzKo0Lx+xt55g9NGM7bhuSB3x0tmbWzRv7EPCBqvrwqOvZnO5rnE8Cx464lNZzgeO7+WNXAC9M8v7RlvRIVXVX9+fdwEfoff09LiaByb4zTlfRG3y143DMHhLH7FlzzB6OsR23DcmDPXZbM+gusrgEuLWq3jzqelpJFiXZu1t+DL2Lfr400qIaVfXaqjqwqhbT+zm8tqr+14jLepgku3cX+dB9HfZiYGyu4K+q/wTuTPKUrukYek/81I7DMXsIHLNnzzF7OMZ53B7oiXs7ss09dnvEZT1Mkr8Dng/sm2QSeF1VXTLaqh7hucCvADd3c8gA/rh7WuM4eDxwWXdl/C7AlVU1lrfrGXP7AR/p/f+VBcDlVfXR0Zb0CL8DfKALUOuBXx1xPRoix+yhcczeOcyHMRvGdNze6W8BJ0mSJLWcbiFJkiQ1DMmSJElSw5AsSZIkNQzJkiRJUsOQLEmSJDUMyRqaJC9LUkl+sq/t0CQ/37f+/CQ/M4v32DvJmX3r+ye5aturnr0kr0ryyhn6LE1y0Wa2/fHcVCZpvnNc3WKfnWJcTbI4yVjd23hnYUjWMJ0C/Ev355RDgZ/vW38+sM2DObA38NBgXlVfq6oTZ3G8Wauqd1TVe2dxiB1mMJc0dI6r28ZxVbNmSNZQJNkDeB5wOr0nD9HdFPw84OVJbkryGuBVwP/u1o/qnqr0oSSrutdzu33PTfLuJJ9Ksj7J73ZvdT7wxG7/v+z/DTvJo5O8J8nNSf49yQu69qVJPpzko0n+I8lfTFP/4Uk+3C2fkOS/kyzsjrm+a39id4wbknxm6sxOV+vZfcdZ01df/2//+7c1JDkfeEzX/wPd05GuTvKFJF9M8vIh/jNJmkccV0czrnb7Tb3+O8nPJvnRJP/Q1fH5JM/o+m6u/dwkl3Wf6fYkv5TkL7q/x4+m90hwkhyW5NPd578myeP72r+Q5AvAbw/6M6Mhqypfvmb9Ak4FLumWPwsc1i0vBS7q63cucHbf+uXA87rlH6f3iNSpfp8FdgP2Be4BHgUsBr7Yt/9D68Af0Hv6FsBPAncAj+5qWA/s1a3fDhzU1L8AWN8tv4neo2+fC/ws8Hdd+yeAQ7rlZ9N7BOnDPhO9x30e2S2f31fbZmsA7uur45eBv+1b32vU/7a+fPkazctxdbTjKvBS4DPd39Ff03tyIsALgZu65c21n0vvG4BHAT8NfBc4rtv2EeBl3bbPAou69pf3/V2vAY7ulv+y/9/H1/Z77fSPpdbQnAK8rVu+olu/YYD9/gewJL1HZgLs2Z09Abi6qr4PfD/J3fQer7klz6M3YFFVX0pyO/DkbtsnqurbAEluAZ4A3Dm1Y/UedfuVJD8FHAG8GTia3mNvP9PV9DPA3/fVulv/myfZG3hsVX2ua7oc+IW+LlusoXMzcGGSC4B/qqrPzPCZJe24HFdHNK4mOYReOH1BVT2Q5Hn0wjZVdW2SfZLs2f39TNcO8M/dvjd3n3nqcdA30/tF5CnA04CPd59/V+Dr3Wfeu6qu6/q/Dzhuppo1fIZkzVqSH6X3G/TTkxS9/9AryR8OsPsuwHOq6nvNMQG+39f0A2b38zrIsa6jNxA9APxf4FJ6n+UPuzq/VVWHzmUNVfXlJM+iN9/wjUk+UVXnzeI9Jc1DjqvDq2Frx9UuvF8J/EZVfX22tVXVg0keqO60MPBgV2eAtVV1ZPP+e8/iPTVEzknWMJwIvK+qnlBVi6vqIOCrwFHAfwGP7evbrn8M+J2plSSHzvBe7f79PkPv60mSPJne14y3Df4x+Azw+8DnqmojsA+93/S/WFXfAb6a5KTu+Eny0/07V9W3gP9K8uyu6eQB3/eBvvlp+wPfrar30zuL8aytqF/SjsNxlbkdV5P8eZJfnGbfdwPvac449/89PB/4Rlf/5toHcRuwKMmR3f6PSvLU7jN/qzt7zdTxtf0ZkjUMp9CbY9XvQ137J+l97XdTd7HEPwK/2K0fBfwuMNFd9HALvQtQNquq7gH+tbv44i+bzX8D7NJ9tfVBYGn3teKgrqf31ePUV1xrgJv7fvs/FTi9u5BiLXDCNMc4HfjbJDcBuwPfHuB9lwNrknwAeDrwb93+rwPeuBX1S9pxOK7+0FyNq08H/rN/pyRPoPcLyq/lhxfvTdCbY3xYkjX05kWf1u2yufYZVdX93Xtd0H3+m/jhXUp+Fbi4qznTHkBzLj/8OZU0W0n2qKr7uuVlwOOr6vdGXJYkzVtzNa4muaaqfm7WBWqH5ZxkabhekuS19P7bup3e1deSpG03J+OqAVkz8UyyJEmS1HBOsiRJktQwJEuSJEkNQ7IkSZLUMCRLkiRJDUOyJEmS1DAkS5IkSY3/D02Tg3aM94A2AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 864x432 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "#Attention weights for one sequence\n",
        "plt.suptitle('Attention weights for one sequence')\n",
        "\n",
        "plt.figure(figsize=(12, 6))\n",
        "a1 = plt.subplot(1, 2, 1)\n",
        "plt.bar(range(len(attention_slice)), attention_slice)\n",
        "# freeze the xlim\n",
        "plt.xlim(plt.xlim())\n",
        "plt.xlabel('Attention weights')\n",
        "\n",
        "a2 = plt.subplot(1, 2, 2)\n",
        "plt.bar(range(len(attention_slice)), attention_slice)\n",
        "plt.xlabel('Attention weights, zoomed')\n",
        "\n",
        "# zoom in\n",
        "top = max(a1.get_ylim())\n",
        "zoom = 0.85*top\n",
        "a2.set_ylim([0.90*top, top])\n",
        "a1.plot(a1.get_xlim(), [zoom, zoom], color='k')"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "52418b41",
      "metadata": {
        "id": "52418b41"
      },
      "source": [
        "**The decoder**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ae2b1529",
      "metadata": {
        "id": "ae2b1529"
      },
      "outputs": [],
      "source": [
        "class Decoder(tf.keras.layers.Layer):\n",
        "  def __init__(self, output_vocab_size, embedding_dim, dec_units):\n",
        "    super(Decoder, self).__init__()\n",
        "    self.dec_units = dec_units\n",
        "    self.output_vocab_size = output_vocab_size\n",
        "    self.embedding_dim = embedding_dim\n",
        "\n",
        "    # For Step 1. The embedding layer convets token IDs to vectors\n",
        "    self.embedding = tf.keras.layers.Embedding(self.output_vocab_size,\n",
        "                                               embedding_dim)\n",
        "\n",
        "    # For Step 2. The RNN keeps track of what's been generated so far.\n",
        "    self.gru = tf.keras.layers.GRU(self.dec_units,\n",
        "                                   return_sequences=True,\n",
        "                                   return_state=True,\n",
        "                                   recurrent_initializer='glorot_uniform')\n",
        "\n",
        "    # For step 3. The RNN output will be the query for the attention layer.\n",
        "    self.attention = BahdanauAttention(self.dec_units)\n",
        "\n",
        "    # For step 4. Eqn. (3): converting `ct` to `at`\n",
        "    self.Wc = tf.keras.layers.Dense(dec_units, activation=tf.math.tanh,\n",
        "                                    use_bias=False)\n",
        "\n",
        "    # For step 5. This fully connected layer produces the logits for each\n",
        "    # output token.\n",
        "    self.fc = tf.keras.layers.Dense(self.output_vocab_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "121d1e5c",
      "metadata": {
        "id": "121d1e5c"
      },
      "outputs": [],
      "source": [
        "class DecoderInput(typing.NamedTuple):\n",
        "  new_tokens: Any\n",
        "  enc_output: Any\n",
        "  mask: Any\n",
        "\n",
        "class DecoderOutput(typing.NamedTuple):\n",
        "  logits: Any\n",
        "  attention_weights: Any"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "73f90e89",
      "metadata": {
        "id": "73f90e89"
      },
      "source": [
        "**Implementation of the call method**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f2a7c966",
      "metadata": {
        "id": "f2a7c966"
      },
      "outputs": [],
      "source": [
        "def call(self,\n",
        "         inputs: DecoderInput,\n",
        "         state=None) -> Tuple[DecoderOutput, tf.Tensor]:\n",
        "  shape_checker = ShapeChecker()\n",
        "  shape_checker(inputs.new_tokens, ('batch', 't'))\n",
        "  shape_checker(inputs.enc_output, ('batch', 's', 'enc_units'))\n",
        "  shape_checker(inputs.mask, ('batch', 's'))\n",
        "\n",
        "  if state is not None:\n",
        "    shape_checker(state, ('batch', 'dec_units'))\n",
        "\n",
        "  # Step 1. Lookup the embeddings\n",
        "  vectors = self.embedding(inputs.new_tokens)\n",
        "  shape_checker(vectors, ('batch', 't', 'embedding_dim'))\n",
        "\n",
        "  # Step 2. Process one step with the RNN\n",
        "  rnn_output, state = self.gru(vectors, initial_state=state)\n",
        "\n",
        "  shape_checker(rnn_output, ('batch', 't', 'dec_units'))\n",
        "  shape_checker(state, ('batch', 'dec_units'))\n",
        "\n",
        "  # Step 3. Use the RNN output as the query for the attention over the\n",
        "  # encoder output.\n",
        "  context_vector, attention_weights = self.attention(\n",
        "      query=rnn_output, value=inputs.enc_output, mask=inputs.mask)\n",
        "  shape_checker(context_vector, ('batch', 't', 'dec_units'))\n",
        "  shape_checker(attention_weights, ('batch', 't', 's'))\n",
        "\n",
        "  # Step 4. Eqn. (3): Join the context_vector and rnn_output\n",
        "  #     [ct; ht] shape: (batch t, value_units + query_units)\n",
        "  context_and_rnn_output = tf.concat([context_vector, rnn_output], axis=-1)\n",
        "\n",
        "  # Step 4. Eqn. (3): `at = tanh(Wc@[ct; ht])`\n",
        "  attention_vector = self.Wc(context_and_rnn_output)\n",
        "  shape_checker(attention_vector, ('batch', 't', 'dec_units'))\n",
        "\n",
        "  # Step 5. Generate logit predictions:\n",
        "  logits = self.fc(attention_vector)\n",
        "  shape_checker(logits, ('batch', 't', 'output_vocab_size'))\n",
        "\n",
        "  return DecoderOutput(logits, attention_weights), state"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "cc7056d1",
      "metadata": {
        "id": "cc7056d1"
      },
      "outputs": [],
      "source": [
        "Decoder.call = call"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "9ccf085c",
      "metadata": {
        "id": "9ccf085c"
      },
      "source": [
        "Using this decoder"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d71d0a3e",
      "metadata": {
        "id": "d71d0a3e"
      },
      "outputs": [],
      "source": [
        "decoder = Decoder(output_text_processor.vocabulary_size(),\n",
        "                  embedding_dim, units)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "bdd8d324",
      "metadata": {
        "id": "bdd8d324"
      },
      "outputs": [],
      "source": [
        "# Convert the target sequence, and collect the \"[START]\" tokens\n",
        "example_output_tokens = output_text_processor(example_target_batch)\n",
        "\n",
        "start_index = output_text_processor.get_vocabulary().index('[START]')\n",
        "first_token = tf.constant([[start_index]] * example_output_tokens.shape[0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9213add8",
      "metadata": {
        "id": "9213add8",
        "outputId": "053f6de9-918f-4f09-9f72-c7519c54f305"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "logits shape: (batch_size, t, output_vocab_size) (64, 1, 3672)\n",
            "state shape: (batch_size, dec_units) (64, 1024)\n"
          ]
        }
      ],
      "source": [
        "# Run the decoder\n",
        "dec_result, dec_state = decoder(\n",
        "    inputs = DecoderInput(new_tokens=first_token,\n",
        "                          enc_output=example_enc_output,\n",
        "                          mask=(example_tokens != 0)),\n",
        "    state = example_enc_state\n",
        ")\n",
        "\n",
        "print(f'logits shape: (batch_size, t, output_vocab_size) {dec_result.logits.shape}')\n",
        "print(f'state shape: (batch_size, dec_units) {dec_state.shape}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ec81fa67",
      "metadata": {
        "id": "ec81fa67"
      },
      "outputs": [],
      "source": [
        "#Sample a token according to the logits\n",
        "sampled_token = tf.random.categorical(dec_result.logits[:, 0, :], num_samples=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c97806c5",
      "metadata": {
        "id": "c97806c5",
        "outputId": "fc553690-cddc-4a5b-e9d9-096910fe9476"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([['beings'],\n",
              "       ['colorful'],\n",
              "       ['goodbye'],\n",
              "       ['returned'],\n",
              "       ['lonely']], dtype='<U19')"
            ]
          },
          "execution_count": 36,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Decode the token as the first word of the output\n",
        "vocab = np.array(output_text_processor.get_vocabulary())\n",
        "first_word = vocab[sampled_token.numpy()]\n",
        "first_word[:5]"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "597267e4",
      "metadata": {
        "id": "597267e4"
      },
      "source": [
        "**Use the decoder to generate a second set of logits**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2641dc6a",
      "metadata": {
        "id": "2641dc6a",
        "outputId": "99840ad8-52b8-467f-9d13-cb2e9eec7992"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([['picked'],\n",
              "       ['rubbed'],\n",
              "       ['misfired'],\n",
              "       ['revenues'],\n",
              "       ['talent']], dtype='<U19')"
            ]
          },
          "execution_count": 37,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "dec_result, dec_state = decoder(\n",
        "    DecoderInput(sampled_token,\n",
        "                 example_enc_output,\n",
        "                 mask=(example_tokens != 0)),\n",
        "    state=dec_state)\n",
        "sampled_token = tf.random.categorical(dec_result.logits[:, 0, :], num_samples=1)\n",
        "first_word = vocab[sampled_token.numpy()]\n",
        "first_word[:5]"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "3d2b14b8",
      "metadata": {
        "id": "3d2b14b8"
      },
      "source": [
        "### Training"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "b2924382",
      "metadata": {
        "id": "b2924382"
      },
      "source": [
        "**Loss function**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "03125e3a",
      "metadata": {
        "id": "03125e3a"
      },
      "outputs": [],
      "source": [
        "class MaskedLoss(tf.keras.losses.Loss):\n",
        "  def __init__(self):\n",
        "    self.name = 'masked_loss'\n",
        "    self.loss = tf.keras.losses.SparseCategoricalCrossentropy(\n",
        "        from_logits=True, reduction='none')\n",
        "\n",
        "  def __call__(self, y_true, y_pred):\n",
        "    shape_checker = ShapeChecker()\n",
        "    shape_checker(y_true, ('batch', 't'))\n",
        "    shape_checker(y_pred, ('batch', 't', 'logits'))\n",
        "\n",
        "    # Calculate the loss for each item in the batch.\n",
        "    loss = self.loss(y_true, y_pred)\n",
        "    shape_checker(loss, ('batch', 't'))\n",
        "\n",
        "    # Mask off the losses on padding.\n",
        "    mask = tf.cast(y_true != 0, tf.float32)\n",
        "    shape_checker(mask, ('batch', 't'))\n",
        "    loss *= mask\n",
        "\n",
        "    # Return the total.\n",
        "    return tf.reduce_sum(loss)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "a493e44b",
      "metadata": {
        "id": "a493e44b"
      },
      "source": [
        "**Implement the training step**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "18eace7d",
      "metadata": {
        "id": "18eace7d"
      },
      "outputs": [],
      "source": [
        "class TrainTranslator(tf.keras.Model):\n",
        "  def __init__(self, embedding_dim, units,\n",
        "               input_text_processor,\n",
        "               output_text_processor, \n",
        "               use_tf_function=True):\n",
        "    super().__init__()\n",
        "    # Build the encoder and decoder\n",
        "    encoder = Encoder(input_text_processor.vocabulary_size(),\n",
        "                      embedding_dim, units)\n",
        "    decoder = Decoder(output_text_processor.vocabulary_size(),\n",
        "                      embedding_dim, units)\n",
        "\n",
        "    self.encoder = encoder\n",
        "    self.decoder = decoder\n",
        "    self.input_text_processor = input_text_processor\n",
        "    self.output_text_processor = output_text_processor\n",
        "    self.use_tf_function = use_tf_function\n",
        "    self.shape_checker = ShapeChecker()\n",
        "\n",
        "  def train_step(self, inputs):\n",
        "    self.shape_checker = ShapeChecker()\n",
        "    if self.use_tf_function:\n",
        "      return self._tf_train_step(inputs)\n",
        "    else:\n",
        "      return self._train_step(inputs)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "94a1232b",
      "metadata": {
        "id": "94a1232b"
      },
      "source": [
        "The _preprocess will:\n",
        "- Receive a batch of input_text, target_text from the tf.data.Dataset.\n",
        "- Convert those raw text inputs to token-embeddings and masks."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "090c3ae7",
      "metadata": {
        "id": "090c3ae7"
      },
      "outputs": [],
      "source": [
        "def _preprocess(self, input_text, target_text):\n",
        "  self.shape_checker(input_text, ('batch',))\n",
        "  self.shape_checker(target_text, ('batch',))\n",
        "\n",
        "  # Convert the text to token IDs\n",
        "  input_tokens = self.input_text_processor(input_text)\n",
        "  target_tokens = self.output_text_processor(target_text)\n",
        "  self.shape_checker(input_tokens, ('batch', 's'))\n",
        "  self.shape_checker(target_tokens, ('batch', 't'))\n",
        "\n",
        "  # Convert IDs to masks.\n",
        "  input_mask = input_tokens != 0\n",
        "  self.shape_checker(input_mask, ('batch', 's'))\n",
        "\n",
        "  target_mask = target_tokens != 0\n",
        "  self.shape_checker(target_mask, ('batch', 't'))\n",
        "\n",
        "  return input_tokens, input_mask, target_tokens, target_mask"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8412355a",
      "metadata": {
        "id": "8412355a"
      },
      "outputs": [],
      "source": [
        "TrainTranslator._preprocess = _preprocess"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "a9a9fea3",
      "metadata": {
        "id": "a9a9fea3"
      },
      "source": [
        "The _train_step will:\n",
        "- Run the encoder on the input_tokens to get the encoder_output and encoder_state.\n",
        "- Initialize the decoder state and loss.\n",
        "- Loop over the target_tokens:\n",
        "- Calculate the gradient of the loss and use the optimizer to apply updates to the model's trainable_variables."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ed69981d",
      "metadata": {
        "id": "ed69981d"
      },
      "outputs": [],
      "source": [
        "def _train_step(self, inputs):\n",
        "  input_text, target_text = inputs  \n",
        "\n",
        "  (input_tokens, input_mask,\n",
        "   target_tokens, target_mask) = self._preprocess(input_text, target_text)\n",
        "\n",
        "  max_target_length = tf.shape(target_tokens)[1]\n",
        "\n",
        "  with tf.GradientTape() as tape:\n",
        "    # Encode the input\n",
        "    enc_output, enc_state = self.encoder(input_tokens)\n",
        "    self.shape_checker(enc_output, ('batch', 's', 'enc_units'))\n",
        "    self.shape_checker(enc_state, ('batch', 'enc_units'))\n",
        "\n",
        "    # Initialize the decoder's state to the encoder's final state.\n",
        "    # This only works if the encoder and decoder have the same number of\n",
        "    # units.\n",
        "    dec_state = enc_state\n",
        "    loss = tf.constant(0.0)\n",
        "\n",
        "    for t in tf.range(max_target_length-1):\n",
        "      # Pass in two tokens from the target sequence:\n",
        "      # 1. The current input to the decoder.\n",
        "      # 2. The target for the decoder's next prediction.\n",
        "      new_tokens = target_tokens[:, t:t+2]\n",
        "      step_loss, dec_state = self._loop_step(new_tokens, input_mask,\n",
        "                                             enc_output, dec_state)\n",
        "      loss = loss + step_loss\n",
        "\n",
        "    # Average the loss over all non padding tokens.\n",
        "    average_loss = loss / tf.reduce_sum(tf.cast(target_mask, tf.float32))\n",
        "\n",
        "  # Apply an optimization step\n",
        "  variables = self.trainable_variables \n",
        "  gradients = tape.gradient(average_loss, variables)\n",
        "  self.optimizer.apply_gradients(zip(gradients, variables))\n",
        "\n",
        "  # Return a dict mapping metric names to current value\n",
        "  return {'batch_loss': average_loss}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "00967a5f",
      "metadata": {
        "id": "00967a5f"
      },
      "outputs": [],
      "source": [
        "TrainTranslator._train_step = _train_step"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "4a3c89d8",
      "metadata": {
        "id": "4a3c89d8"
      },
      "source": [
        "The _loop_step method, added below, executes the decoder and calculates the incremental loss and new decoder state (dec_state)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c893acc6",
      "metadata": {
        "id": "c893acc6"
      },
      "outputs": [],
      "source": [
        "def _loop_step(self, new_tokens, input_mask, enc_output, dec_state):\n",
        "  input_token, target_token = new_tokens[:, 0:1], new_tokens[:, 1:2]\n",
        "\n",
        "  # Run the decoder one step.\n",
        "  decoder_input = DecoderInput(new_tokens=input_token,\n",
        "                               enc_output=enc_output,\n",
        "                               mask=input_mask)\n",
        "\n",
        "  dec_result, dec_state = self.decoder(decoder_input, state=dec_state)\n",
        "  self.shape_checker(dec_result.logits, ('batch', 't1', 'logits'))\n",
        "  self.shape_checker(dec_result.attention_weights, ('batch', 't1', 's'))\n",
        "  self.shape_checker(dec_state, ('batch', 'dec_units'))\n",
        "\n",
        "  # `self.loss` returns the total for non-padded tokens\n",
        "  y = target_token\n",
        "  y_pred = dec_result.logits\n",
        "  step_loss = self.loss(y, y_pred)\n",
        "\n",
        "  return step_loss, dec_state"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "fc1af270",
      "metadata": {
        "id": "fc1af270"
      },
      "outputs": [],
      "source": [
        "TrainTranslator._loop_step = _loop_step"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "d325f697",
      "metadata": {
        "id": "d325f697"
      },
      "source": [
        "**Test the training step**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d2057aea",
      "metadata": {
        "id": "d2057aea"
      },
      "outputs": [],
      "source": [
        "translator = TrainTranslator(\n",
        "    embedding_dim, units,\n",
        "    input_text_processor=input_text_processor,\n",
        "    output_text_processor=output_text_processor,\n",
        "    use_tf_function=False)\n",
        "\n",
        "# Configure the loss and optimizer\n",
        "translator.compile(\n",
        "    optimizer=tf.optimizers.Adam(),\n",
        "    loss=MaskedLoss(),\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8faae06f",
      "metadata": {
        "id": "8faae06f"
      },
      "outputs": [],
      "source": [
        "@tf.function(input_signature=[[tf.TensorSpec(dtype=tf.string, shape=[None]),\n",
        "                               tf.TensorSpec(dtype=tf.string, shape=[None])]])\n",
        "def _tf_train_step(self, inputs):\n",
        "  return self._train_step(inputs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2074ff36",
      "metadata": {
        "id": "2074ff36"
      },
      "outputs": [],
      "source": [
        "TrainTranslator._tf_train_step = _tf_train_step"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5d54187e",
      "metadata": {
        "id": "5d54187e"
      },
      "outputs": [],
      "source": [
        "translator.use_tf_function = True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2d38d2cd",
      "metadata": {
        "id": "2d38d2cd",
        "outputId": "f57ea737-20f7-4aa2-bdf0-295947c8ab3d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=7.389841>}"
            ]
          },
          "execution_count": 50,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "translator.train_step([example_input_batch, example_target_batch])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c3458a8b",
      "metadata": {
        "id": "c3458a8b",
        "outputId": "419cf405-a18c-49bf-f66d-da4f842a6dc7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=7.3574944>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=7.294251>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=7.0991335>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=6.2763376>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=5.3231807>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.7004666>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.6283717>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.4811306>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.3151865>}\n",
            "{'batch_loss': <tf.Tensor: shape=(), dtype=float32, numpy=4.211711>}\n",
            "\n",
            "Wall time: 2min 7s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "for n in range(10):\n",
        "  print(translator.train_step([example_input_batch, example_target_batch]))\n",
        "print()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "f0704a09",
      "metadata": {
        "id": "f0704a09"
      },
      "source": [
        "**Plot the result for loss**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e7d991bc",
      "metadata": {
        "id": "e7d991bc",
        "outputId": "65563955-7fa1-47c5-9f6c-d2cd51db93e1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "....................................................................................................\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x275838533d0>]"
            ]
          },
          "execution_count": 52,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAl+ElEQVR4nO3dd3hUVf7H8fc3vRAIkNBL6FVqhCCICKKIhXXVXRRBURdXxbrlp27Xbe7aRUFERSzYd0WsIFKlhV5C7xAghN5CQs7vj4wuxgBDMsnNTD6v55mHmbknd77XCx9vzpx7jjnnEBGR4BfmdQEiIhIYCnQRkRChQBcRCREKdBGREKFAFxEJERFefXBSUpJLSUnx6uNFRILSggUL9jjnkova5lmgp6SkkJ6e7tXHi4gEJTPbfLpt6nIREQkRCnQRkRChQBcRCREKdBGREKFAFxEJEQp0EZEQoUAXEQkRno1DL641uw4xcckOoiLCiAwPo3JsJD/pUJfYqHCvSxMR8VRQBvpzU9b94L1xszcz6qZONKwe71FVIiLeM68WuEhNTXXFvVPUOUdeviP3ZD6z12fz4HtLcM7xzMAO9G5ZM8CVioiUH2a2wDmXWtS2oOxDNzMiw8OIi4qgT6uaTLynB/WrxXHr2HS+WrHT6/JERDwRlIFeWP1qcXx45wW0rJXAHz9ewaHjuV6XJCJS5kIi0AFiIsP557Xt2HXoOE9+tcbrckREylzIBDpAh/qJDElryOuzN7F4636vyxERKVN+B7qZhZvZIjObWMS2aDN718zWmdlcM0sJaJXn4NeXtaBmQgwPf7SM3JP5XpUhIlLmzuUK/T4g4zTbbgP2OeeaAk8Dj5e0sOJKiInkz1e3ISPzIK9/u8mrMkREypxfgW5m9YArgDGnaTIAeN33/AOgj5lZycsrnn5ta3FR82Se/Xot2YdzvCpDRKRM+XuF/gzwW+B0fRh1ga0Azrk84ABQvaTFlcTvr2jF0RMneWqSviAVkYrhrIFuZlcCu51zC0r6YWY2zMzSzSw9KyurpLs7o2Y1Exic1pDx87awaufBUv0sEZHywJ8r9O7A1Wa2CXgH6G1mbxZqsx2oD2BmEUAVILvwjpxzo51zqc651OTkItc4Daj7L2lG5dhIHpu4Eq/uiBURKStnncvFOfcw8DCAmfUCfu2cu6lQswnAzcBs4DpgiisHCZoYF8UDlzTnTxNW0O+ZGeTl55OTl49zEBFuhIcZt1yQwpBuKV6XKiJSYsWenMvMHgXSnXMTgFeAN8xsHbAXGBig+kpsUNcGZGQeZM/hHKIjwomOCAODk/mOTdlH+dOEFTSsHs9FzUv/NwYRkdIUlJNzBcqxEye55sVZ7Dp4nIn3XkjdxFhP6xEROZuQm5wrUGKjwnlxUCdyTzruemshOXknvS5JRKTYKnSgAzROrsQT17djydb93PLqfCav3EWe7jAVkSAUdAtclIZ+bWvz56taM+Kb9dw+Lp2alaO5un0dLm5Rg9SUakRFVPj/74lIEKjQfeiF5Z7MZ8qq3bw7fysz1maRe9IRHxVO//Nq83D/VlSLj/K6RBGp4M7Uh64r9FNEhodxWZtaXNamFkdy8vh2fTZTVu3igwXb+Gb1bh4b0JbLz6vtdZkiIkVSX8JpxEdH0Ld1Tf7x03Z8ck8PalWJ4c63FnLP+EUczsnzujwRkR9RoPuhZa3K/Peu7vz60uZ8tiyTn7wwi/VZh70uS0TkBxTofooID2N472a8cVsX9h45wU9GzOLTpZmaUkBEyg0F+jm6oEkSn9zTg0bJ8dz99kL6PDmNMTM2kH04R+EuIp7SKJdiysk7ycQlmbw1dzMLt+wHICLMqBwbSd3EWB4d0IaODap6W6SIhJwzjXJRoAfAyh0HmbE2i/3Hcjl4LJdpa7LYdfA4f7yyNTelNcTDtT5EJMRo2GIpa12nMq3rVP7+9YGjuTzw3mL+8PEKFmzex9+uOY/4aP2nFpHSpT70UlAlLpIxQ1L59aXN+XjJDq58fibLtx/wuiwRCXEK9FISFmYM792M8b9I43huwayOL0/fwMl8fXEqIqVDgV7K0hpX5/P7LqR3yxr87bMM+jw5lXfnb+FEniYAE5HAUqCXgcS4KEbd1JlRN3WmUkwE//fhMnr9+xvmb9rrdWkiEkL8WSQ6xszmmdkSM1thZn8pos0tZpZlZot9j9tLp9zgZWb0a1uLT4b3YOzQ84mJDGfIK/OYvf5HS6+KiBSLP1foOUBv51x7oAPQz8zSimj3rnOug+8xJpBFhhIzo1eLGrxzRxr1qsYydOw8Zq3b43VZIhICzhrorsB3E5dE+h76Zq+EaiTEMH5YGg2rxXPr2PlMWrnL65JEJMj51YduZuFmthjYDUxyzs0totm1ZrbUzD4ws/qn2c8wM0s3s/SsrKziVx0ikipFM35YGi1qJTDsjXRenLpO0weISLH5FejOuZPOuQ5APaCLmbUt1OQTIMU51w6YBLx+mv2Mds6lOudSk5OTS1B26KgWH8W7w7pxZbs6/OuL1dz3zmKO52ptUxE5d+c0ysU5tx/4BuhX6P1s51yO7+UYoHNAqqsgYqPCeW5gB35zWQs+WbqD/s/NYMFmjYARkXPjzyiXZDNL9D2PBfoCqwq1OXUZn6uBjADWWCGYGXdf3JQ3bu1KTm4+142azaOfrOTYCV2ti4h//LlCrw18Y2ZLgfkU9KFPNLNHzexqX5t7fUMalwD3AreUTrmhr0ezJL58oCeDujbg1VkbuebFWWzOPuJ1WSISBDTbYjk2bU0W945fBMBzN3Tkoub63kGkojvTbIu6U7Qcu6h5Mp8M70HtKjHc8to8xszY4HVJIlKOKdDLuQbV4/jorgvo16YWf/00g39/uUpDG0WkSAr0IBAXFcGIGztxQ5f6vPDNev748QryNWujiBSiVReCRHiY8fdrzqNybCQvTdvA/mO5PHF9O6Ijwr0uTUTKCQV6EDEzHr68FVXjovjn56vYcyiHUYM7UyU20uvSRKQcUJdLEPrlRU14+uftSd+8l5+Nms2O/ce8LklEygEFepC6pmM9xg7two79x7h+1Gy27j3qdUki4jEFehDr3jSJ8cPSOHQ8lxvHzNGVukgFp0APcm3rVuHN27uy/2guN7w8h50Hjntdkoh4RIEeAtrVS2TcrV3IPnyCgaPV/SJSUSnQQ0THBlUZd1sX9h3N5acjvyUj86DXJYlIGVOgh5BODary/i+7EW7Gz16azdwNWq9UpCJRoIeY5jUT+PCuC6iREM3Nr80jfZPmVRepKBToIahuYizv3tGN2lViuXXsfFbvPOR1SSJSBhToISqpUjTjbu1CbFQ4Q16dqy9KRSoABXoIq18tjnG3duV4bj6Dxsxl2bYDXpckIqXInyXoYsxsnpkt8a1K9Jci2kSb2btmts7M5ppZSqlUK+esRa0Exg49nxN5+Vzz4iyembyG3JP5XpclIqXAnyv0HKC3c6490AHoZ2ZphdrcBuxzzjUFngYeD2iVUiIdG1Tly/t7clX7OjwzeS1XPT+TEVPWsnDLPvIU7iIh46yB7goc9r2M9D0KT8Y9AHjd9/wDoI+ZWcCqlBKrEhfJ0z/vwMhBnQgz44mv1vDTF7+ly9+/ZsnW/V6XJyIB4FcfupmFm9liYDcFi0TPLdSkLrAVwDmXBxwAqhexn2Fmlm5m6VlZWSUqXIrn8vNq89l9F7Lg95fwwo2diIsK57bX57MlW1+aigQ7vwLdOXfSOdcBqAd0MbO2xfkw59xo51yqcy41OVkLHnupeqVormhXm9dv7UJevuOW1+ax78gJr8sSkRI4p1Euzrn9wDdAv0KbtgP1AcwsAqgC6DbFINAkuRJjhqSybf8xbh+XzvHck16XJCLF5M8ol2QzS/Q9jwX6AqsKNZsA3Ox7fh0wxWkl46CRmlKNZ37egYVb9jH87UX6olQkSPlzhV4b+MbMlgLzKehDn2hmj5rZ1b42rwDVzWwd8CDwUOmUK6Wl/3m1+fNVbZicsYvf/Wc5+v+xSPA565qizrmlQMci3v/jKc+PA9cHtjQpazdfkMKewzk8P2UdSQlR/Oayll6XJCLnQItEyw882Lc5ew7n8MI369my9xgPX96SOomxXpclIn5QoMsPmBmPDWhLckIMo6atZ/LKXdzVqwm3X9iY2Khwr8sTkTPQXC7yIxHhYTzYtzlfP3gRvVok8+SkNfR4fAojp67ncE6e1+WJyGko0OW06leLY+RNnXn/l91oU7cKj3+xiu7/nML4eVv0palIOaRAl7M6P6Ua427twsd3d6d17co8/NEy7hm/iEPHc70uTUROoUAXv7Wvn8hbt3flN5e14PPlO7niuZlau1SkHFGgyzkJCzPuvrgp7w5L40RePje8PIeVOxTqIuWBAl2KJTWlGu/d0Y3YyHBuemWulrkTKQcU6FJsDarH8fYv0ogIMwaNmaNQF/GYAl1KpFFSPOOHpWFmXD1iJq/M3Eh+vkbAiHhBgS4l1iS5EhPv6UH3pkk8NnElA1+ew7rdh8/+gyISUAp0CYialWN45eZU/nVdOzJ2HOSSp6Zx05i5fL4sU2uYipQR8+oGkdTUVJeenu7JZ0vpyjqUw7vztzB+3la27z9G46R4/n19ezo3rOp1aSJBz8wWOOdSi9qmK3QJuOSEaIb3bsb0317MS4M7k5OXz/WjvuXxL1aRk6cFNERKiwJdSk14mHFZm1p8cf+FXN+5PiOnrmfAiFms2aXRMCKlQYEupS4hJpLHr2vHq7eksudwDlc9P5M3Zm/SfDAiAebPEnT1zewbM1tpZivM7L4i2vQyswNmttj3+GNR+5KKrXfLmnx+X0/SGlfnDx+v4M43F6oLRiSA/LlCzwN+5ZxrDaQBd5tZ6yLazXDOdfA9Hg1olRIykhOiee2W83n48pZ8sWIn945fpFEwIgFy1kB3zmU65xb6nh8CMoC6pV2YhK6wMOOOi5rwp6ta8+WKXfzqvSWc1M1IIiV2TisWmVkKBeuLzi1iczczWwLsAH7tnFtRxM8PA4YBNGjQ4JyLldAytHsjjuWe5F9frCYqIoy/XdOW6AitiiRSXH4HuplVAj4E7nfOFZ5ebyHQ0Dl32Mz6A/8FmhXeh3NuNDAaCsahF7doCR139WpKTm4+z369lozMgzw7sANNayR4XZZIUPJrlIuZRVIQ5m855z4qvN05d9A5d9j3/DMg0sySAlqphKwH+jZn9ODOZB44zpXPz+S1WRs5ekJL3YmcK39GuRjwCpDhnHvqNG1q+dphZl18+80OZKES2i5tU4sv7ruQLo2q85dPVtL5scnc984ipq3J0vBGET/50+XSHRgMLDOzxb73HgEaADjnRgHXAXeaWR5wDBjo9K9QzlGNyjG8PvR85m3cy8dLdvDp0kw+XryDq9rX4W/XtKVyTKTXJYqUa5rLRcqtE3n5vDxjA09NWkOdxBiev6ETHeonel2WiKc0l4sEpaiIMO6+uCnv3ZFGfj5cN/JbPl683euyRMotBbqUe50bVuOz+y4kNaUq97+7mHfmbfG6JJFySYEuQaFKbCRjh3bhoubJPPTRMl6ZudHrkkTKHQW6BI2YyHBeGtyZfm1q8djElYyevt7rkkTKFQW6BJXoiHBG3NiRK9rV5u+freK1WbpSF/nOOd36L1IeRISH8czPO5B3Mp+/fLKSyPAwbkpr6HVZIp7TFboEpcjwMJ6/oRO9W9bg9/9dzhNfruZEnmZtlIpNgS5BKyoijBcHdeK6zvUY8c06Brwwi4zMwtMMiVQcCnQJajGR4TxxfXteHpJK1qEcrh4xk9HT15Ov6XilAlKgS0jo27omXz3Qk94ta/D3z1Zx6+vzyT6c43VZImVKgS4ho1p8FKNu6sxjA9rw7fpsLn92Bumb9npdlkiZUaBLSDEzBndL4b93dSc+OoJBY+YyaeUur8sSKRMKdAlJretU5oNfdqNlrQR++eYC3kvf6nVJIqVOgS4hq3qlaN7+RRoXNKnObz9YqjtLJeQp0CWkxUdH8MrN539/Z+mIKWu9Lkmk1PizYlF9M/vGzFaa2Qozu6+INmZmz5nZOjNbamadSqdckXMXFRHGsz/vwDUd6/LEV2t4atIarYIkIcmfW//zgF855xaaWQKwwMwmOedWntLmcgoWhW4GdAVG+v4UKRciwsN44vr2RIQZz329lqM5eTzcvxXhYeZ1aSIBc9ZAd85lApm+54fMLAOoC5wa6AOAcb5l5+aYWaKZ1fb9rEi5EB5mPH5tO+KjIxgzcyMb9hzh2YEdSNDSdhIizqkP3cxSgI7A3EKb6gKnDiPY5ntPpFwJCzP+fHUbHhvQhmlrsvjpi9+yOfuI12WJBITfgW5mlYAPgfudc8WaMMPMhplZupmlZ2VlFWcXIgExuFsKb9zahd2Hcrjy+Zl8smSH1yWJlJhfgW5mkRSE+VvOuY+KaLIdqH/K63q+937AOTfaOZfqnEtNTk4uTr0iAXNB0yQm3tODpjUqcc/4Rfzm/SUcycnzuiyRYvNnlIsBrwAZzrmnTtNsAjDEN9olDTig/nMJBvWrxfHeHd0YfnFTPli4jT5PTmPMjA0cVrBLELKzDd8ysx7ADGAZ8N2E048ADQCcc6N8oT8C6AccBYY659LPtN/U1FSXnn7GJiJlav6mvTz11Rpmb8gmISaC23s05q6LmxAZrts1pPwwswXOudQit3k1HleBLuXV4q37GTl1HV+u2EX7elV4dmBHUpLivS5LBDhzoOvSQ6SQDvUTeWlwKi8O6sTGPUe44rkZ/HfRj74SEil3FOgip9H/vNp8cX9P2tSpwv3vLubNOZu9LknkjBToImdQJzGWN2/vSh/f2qXvzdesjVJ+KdBFziIqIowXBnXiwmZJ/N9HS/nPom1elyRSJAW6iB9iIsN5eUgqaY2q8+v3lzJnQ7bXJYn8iAJdxE8xkeGMHtKZhtXjGP72QnYeOO51SSI/oEAXOQcJMZG8dFNnjp44yZ1vLSAn7+SP2jjn2LRH88NI2VOgi5yjZjUT+Pd17Vm0ZT9/nrCC47n/C/XN2UcYNGYuvZ6YymStZSplzJ/50EWkkCva1Wbptsa8NH0DExbvoG/rmtSrGsfLMzYQFR5G1bhIXp+9iUta1/S6VKlAFOgixfTQ5S25sFkynyzZwRcrdnLgWC6Xtq7JowPa8l76Vp6atIZNe47oLlMpM7r1XyQATuTls/PAcRpUjwNg98HjXPDPKdzaoxGP9G/lcXUSSnTrv0gpi4oI+z7MAWpUjuHSNjV5P33rD/rYRUqTAl2klAzq2pB9R3P5fLlmkpayoUAXKSUXNKlO46R43pyzhfx8x/qsw3y+LFOLaEip0ZeiIqXEzLixawP++mkG7f/yFYd8QX5j1wb8/ZrzPK5OQpECXaQU/ez8+szftJfkhGja10tk3sa9vDNvC4PTGtKqdmWvy5MQc9ZAN7NXgSuB3c65tkVs7wV8DGz0vfWRc+7RANYoErQqx0Ty0uD/DUjo27omkzJ28ddPV/LmbV0pWOxLJDD86UMfS8HScmcywznXwfdQmIucRmJcFA9c0pxZ67KZnLHb63IkxJw10J1z04G9ZVCLSIVwY9cGNK1Rib99upITefln/wERPwVqlEs3M1tiZp+bWZvTNTKzYWaWbmbpWVlZAfpokeASGR7G769oxabsowx/eyE79h/zuiQJEYEI9IVAQ+dce+B54L+na+icG+2cS3XOpSYnJwfgo0WCU68WNfhtvxZMW5NFnyenMWLKWt2AJCVW4kB3zh10zh32Pf8MiDSzpBJXJhLi7urVlMkPXkSvFsk88dUafjEundyT6oKR4itxoJtZLfN9VW9mXXz71HIuIn6oXy2OkTd15h8/PY8Za/fwu/8sw6v5lST4+TNscTzQC0gys23An4BIAOfcKOA64E4zywOOAQOd/kaKnJMbujQgc/8xnpuyjvpV47inTzOvS5IgdNZAd87dcJbtI4ARAatIpIJ6oG9ztu07xpOT1hAebtzWoxHREeFelyVBRHO5iJQTZsY/r21H39Y1+dcXq+nz5DQ+WriNk/n6hVf8o0AXKUeiIsIYPbgz427tQpXYSB58bwnDxqWTr1AXPyjQRcoZM6Nn82Q+Gd6Dhy9vyderdvPi1HVelyVBQIEuUk6FhRnDejbm6vZ1eGrSGr5dv8frkqScU6CLlGNmxt9/eh4pSfHcO34xuw8d97okKccU6CLlXKXoCF4c1InDOblcN3I2L0/fQPbhHK/LknJIgS4SBFrWqszowalUrxTF3z7LIO0fX/PnCSt0E5L8gBa4EAkSPZsn07N5Mmt2HWL09A2M/XYTjZPjGdItxevSpJzQFbpIkGleM4F/XduOXi2S+evEDFbsOOB1SVJOKNBFglBYmPHk9e2pGh/JPW8v0sLTAijQRYJW9UrRPDuwI5uyj3DfO4tYs+uQ1yWJxxToIkEsrXF1Hr68FVNXZ3Hp09O56vmZvDNvi74sraAU6CJB7hc9GzPnkT784crW5OU7HvpoGcPfXsTRE+qGqWgU6CIhIKlSNLf1aMRn9xZMF/DZ8kyuHTmbrXuPel2alCENWxQJIWbGHRc1oUWtBO4Zv4g+T02jYbU46laNpVmNSgzv3YwqsZFelymlRFfoIiGoV4safDK8B0PSGtI4OZ49h3N4bdYmrh35ra7aQ5id7csTM3sVuBLY7ZxrW8R2A54F+gNHgVuccwvP9sGpqakuPT29WEWLyLmbvT6bX765gIgwY/SQVDo3rOp1SVIMZrbAOZda1DZ/rtDHAv3OsP1yoJnvMQwYea4Fikjp69akOh/ddQGVYiK44eU5fLtOszeGmrMGunNuOrD3DE0GAONcgTlAopnVDlSBIhI4TZIr8Z+7upNSPY473lhARuZBr0uSAApEH3pdYOspr7f53vsRMxtmZulmlp6VlRWAjxaRc1UtPoqxQ7sQFx3O0Nfms2P/Ma9LkgAp0y9FnXOjnXOpzrnU5OTksvxoETlFncRYxg7twpGcPG55bR4HjuV6XZIEQCACfTtQ/5TX9XzviUg51qp2ZV4a3JkNWUf41XtLtG5pCAhEoE8AhliBNOCAcy4zAPsVkVJ2QdMkHunfiskZuxg1fb3X5UgJnfXGIjMbD/QCksxsG/AnIBLAOTcK+IyCIYvrKBi2OLS0ihWRwBvaPYWFW/bxxJeraV8vke5Nk3DOkXU4h+RK0RSMTJZgcNZx6KVF49BFyo8jOXkMeGEWe4+coE2dyizffoB9R3MZ1rMxj/Rv5XV5coqSjkMXkRAXHx3BqJs6ExsZzt4jJ7isTS36tq7J6OkbmLp6t9fliZ90hS4iRTqee5IBI2aRfSSHz+/rSXJCtNclCbpCF5FiiIkM5/kbO3LoeB6/el+jYIKBAl1ETqt5zQT+cGVrpq/J4q63FrJ8u9YvLc80fa6InNGgrg3YfSiHV2Zs4IsVO+nRNIkH+jajc8NqXpcmhagPXUT8cuBYLm/P3cKrszaSdSiHG7o04KF+LakSp/nVy5L60EWkxKrERnJnryZM/XUvfnFhI95L30qfp6by2TLdR1heKNBF5JzER0fwuyta8/Hd3amTGMtdby3koQ+Xag3TckCBLiLF0rZuFT688wLu6tWEd9O3ctXzM1m0ZZ/XZVVoCnQRKbbI8DB+268lb97WlUPH87jmxW8ZOHo2k1fu0jBHDyjQRaTEujdNYvKvLuKR/i3Zkn2U28el0/+5GaRvOtPaOBJoCnQRCYjKMZEM69mEab+9mGcHduDgsVyuGzWbhz5cyr4jJ7wur0JQoItIQEWGhzGgQ10mPXgRw3o25v0F27j4yam8OnMjJ/LyvS4vpCnQRaRUxEdH8Ej/Vky8pwdt6lTm0YkrufTpaXy6NFP966VEgS4ipapV7cq8eVtXXht6PpHhYdz99kL6Pj2N99K36oo9wHSnqIiUmbyT+Xy+fCcjp65nZeZBaleJ4c5eTfhZan1iIsO9Li8olPhOUTPrZ2arzWydmT1UxPZbzCzLzBb7HreXtGgRCT0R4WFc1b4On97bg7FDz6duYix//HgFvf49lbGzNurmpBI66xW6mYUDa4C+wDZgPnCDc27lKW1uAVKdc8P9/WBdoYuIc45v12fz7OS1zNu0l8S4SAZ1bcCQbinUrBzjdXnl0pmu0P2ZbbELsM45t8G3s3eAAcDKM/6UiMhZmBndmyZxQZPqLNi8jzEzNvLi1PWMnr6Bq9vX5fYLG9GqdmWvywwa/gR6XWDrKa+3AV2LaHetmfWk4Gr+Aefc1sINzGwYMAygQYMG516tiIQkMyM1pRqpKdXYnH2E12Zt4r30rXy4cBs9mibx00516dOqJlViNbPjmfjT5XId0M85d7vv9WCg66ndK2ZWHTjsnMsxszuAnzvnep9pv+pyEZEzOXA0l7fmbebN2ZvZceA4keEFV/M3dW1I75Y1CAszr0v0xJm6XPwJ9G7An51zl/lePwzgnPvHadqHA3udc1XOtF8Fuoj4Iz/fsWTbfj5fvpNPluwg88BxmiTHc0fPJlzRrjbx0RVrnZ6SBnoEBd0ofYDtFHwpeqNzbsUpbWo75zJ9z68B/s85l3am/SrQReRc5Z7M57NlmYyatoGMzINEhYfRtXE1+rSsweXn1a4QX6SWKNB9O+gPPAOEA6865/5mZo8C6c65CWb2D+BqIA/YC9zpnFt1pn0q0EWkuJxzzNu4l0krdzFl9W42ZB0hPMzo1TyZn51fn4tb1CAqIjTvmyxxoJcGBbqIBMr6rMN8sGAbHyzYRtahHBKiI+jZIpm+rWpycYsaIbVMngJdRCqEvJP5TF+bxZfLd/H1qt3sOZxDeJiR1rgal7auRZ9WNahXNc7rMktEgS4iFc53X6Z+tXIXX63YyfqsIwCkVI+jR7MkejZLpkezJOKigutLVQW6iFR463YfZvqaLGau28OcDdkcPXGS6IgwujdNoleLZDo1qEqLWglEhpfvvveS3ikqIhL0mtaoRNMalbi1RyNO5OUzf9NeJmfsYnLGLqas2g1ATGQYHeon0rN5Mr2a16BV7QTMgme8u67QRaRCc86xff8xFm3Zz6It+5mzIZuVmQcBqB4fRbt6VTivbhXOq5dIxwaJJFWK9rReXaGLiJyGmVGvahz1qsZxVfs6AOw+eJxpa7KYvSGb5dsPMG1NFt+tydGgWhydGiTSrl4i7epVoXWdyuWmH15X6CIiZ3H0RB4rdhxk4eZ9LNyyj0Vb9rP7UA4AYQaNkuJpU6cg3FvXrkyr2pVJTiidK3ldoYuIlEBcVATnp1Tj/JRq37+36+Bxlm07wNLtB8jIPMiCzfuYsGTH99uTKkXTvGal7/vum9VIoGWtBKrGR5VanQp0EZFiqFk5hpqtY7ikdc3v39t/9AQZmYdYmXmQjMyDrN19mI8Wbudwzv8W7qiREM2wno25/cLGAa9JgS4iEiCJcVF0a1Kdbk2qf/+ec45dB3NYvesQq3ceZNXOQ6XWHaNAFxEpRWZGrSox1KoSw0XNk0v1s8r3CHoREfGbAl1EJEQo0EVEQoQCXUQkRCjQRURChF+Bbmb9zGy1ma0zs4eK2B5tZu/6ts81s5SAVyoiImd01kD3Lfr8AnA50Bq4wcxaF2p2G7DPOdcUeBp4PNCFiojImflzhd4FWOec2+CcOwG8Awwo1GYA8Lrv+QdAHwumOSdFREKAPzcW1QW2nvJ6G9D1dG2cc3lmdgCoDuw5tZGZDQOG+V4eNrPVxSkaSCq87wqiIh53RTxmqJjHXRGPGc79uBuebkOZ3inqnBsNjC7pfsws/XSzjYWyinjcFfGYoWIed0U8ZgjscfvT5bIdqH/K63q+94psY2YRQBUgOxAFioiIf/wJ9PlAMzNrZGZRwEBgQqE2E4Cbfc+vA6Y4ryZaFxGpoM7a5eLrEx8OfAmEA68651aY2aNAunNuAvAK8IaZrQP2UhD6panE3TZBqiIed0U8ZqiYx10RjxkCeNyerVgkIiKBpTtFRURChAJdRCREBF2gn20aglBgZvXN7BszW2lmK8zsPt/71cxskpmt9f1Z1etaS4OZhZvZIjOb6HvdyDelxDrfFBOltyijB8ws0cw+MLNVZpZhZt0qwrk2swd8f7+Xm9l4M4sJxXNtZq+a2W4zW37Ke0WeXyvwnO/4l5pZp3P5rKAKdD+nIQgFecCvnHOtgTTgbt9xPgR87ZxrBnztex2K7gMyTnn9OPC0b2qJfRRMNRFKngW+cM61BNpTcOwhfa7NrC5wL5DqnGtLwYCLgYTmuR4L9Cv03unO7+VAM99jGDDyXD4oqAId/6YhCHrOuUzn3ELf80MU/AOvyw+nWHgd+IknBZYiM6sHXAGM8b02oDcFU0pAiB23mVUBelIwUgzn3Ann3H4qwLmmYJRdrO/elTggkxA818656RSM/jvV6c7vAGCcKzAHSDSz2v5+VrAFelHTENT1qJYy4Zu5siMwF6jpnMv0bdoJ1DzdzwWxZ4DfAvm+19WB/c6575ZND7Vz3gjIAl7zdTONMbN4QvxcO+e2A08AWygI8gPAAkL7XJ/qdOe3RBkXbIFeoZhZJeBD4H7n3MFTt/lu3AqpMadmdiWw2zm3wOtaylAE0AkY6ZzrCByhUPdKiJ7rqhRcjTYC6gDx/LhbokII5PkNtkD3ZxqCkGBmkRSE+VvOuY98b+/67tcv35+7vaqvlHQHrjazTRR0p/WmoH850fdrOYTeOd8GbHPOzfW9/oCCgA/1c30JsNE5l+WcywU+ouD8h/K5PtXpzm+JMi7YAt2faQiCnq/f+BUgwzn31CmbTp1i4Wbg47KurTQ55x52ztVzzqVQcG6nOOcGAd9QMKUEhNhxO+d2AlvNrIXvrT7ASkL8XFPQ1ZJmZnG+v+/fHXfInutCTnd+JwBDfKNd0oADp3TNnJ1zLqgeQH9gDbAe+J3X9ZTSMfag4FewpcBi36M/Bf3JXwNrgclANa9rLcX/Br2Aib7njYF5wDrgfSDa6/oCfKwdgHTf+f4vULUinGvgL8AqYDnwBhAdiucaGE/B9wS5FPxGdtvpzi9gFIzkWw8so2AUkN+fpVv/RURCRLB1uYiIyGko0EVEQoQCXUQkRCjQRURChAJdRCREKNBFREKEAl1EJET8PyFS75x//pvOAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "losses = []\n",
        "for n in range(100):\n",
        "  print('.', end='')\n",
        "  logs = translator.train_step([example_input_batch, example_target_batch])\n",
        "  losses.append(logs['batch_loss'].numpy())\n",
        "\n",
        "print()\n",
        "plt.plot(losses)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "53f12728",
      "metadata": {
        "id": "53f12728"
      },
      "source": [
        "We can confirm that the train step is working"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "8546a422",
      "metadata": {
        "id": "8546a422"
      },
      "source": [
        "**Build a fresh copy of the model to train from scratch**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e4221577",
      "metadata": {
        "id": "e4221577"
      },
      "outputs": [],
      "source": [
        "train_translator = TrainTranslator(\n",
        "    embedding_dim, units,\n",
        "    input_text_processor=input_text_processor,\n",
        "    output_text_processor=output_text_processor)\n",
        "\n",
        "# Configure the loss and optimizer\n",
        "train_translator.compile(\n",
        "    optimizer=tf.optimizers.Adam(),\n",
        "    loss=MaskedLoss(),\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "07f30d05",
      "metadata": {
        "id": "07f30d05"
      },
      "source": [
        "### Train the model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "cf307f2d",
      "metadata": {
        "id": "cf307f2d"
      },
      "outputs": [],
      "source": [
        "class BatchLogs(tf.keras.callbacks.Callback):\n",
        "  def __init__(self, key):\n",
        "    self.key = key\n",
        "    self.logs = []\n",
        "\n",
        "  def on_train_batch_end(self, n, logs):\n",
        "    self.logs.append(logs[self.key])\n",
        "\n",
        "batch_loss = BatchLogs('batch_loss')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c76f7efb",
      "metadata": {
        "id": "c76f7efb",
        "outputId": "cb696caf-bee8-471a-c7b8-55dc83709cd8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "119/119 [==============================] - 729s 6s/step - batch_loss: 4.4652\n",
            "Epoch 2/3\n",
            "119/119 [==============================] - 625s 5s/step - batch_loss: 3.4841\n",
            "Epoch 3/3\n",
            "119/119 [==============================] - 735s 6s/step - batch_loss: 2.8791\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x27583878a60>"
            ]
          },
          "execution_count": 55,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_translator.fit(dataset, epochs=3,\n",
        "                     callbacks=[batch_loss])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "90e2e2ba",
      "metadata": {
        "id": "90e2e2ba",
        "outputId": "b2b842d9-2d7c-4475-eb13-c97f96f83ae3"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Text(0, 0.5, 'CE/token')"
            ]
          },
          "execution_count": 56,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEKCAYAAAAVaT4rAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAxEklEQVR4nO3dd3xV9fnA8c+Tm0UWSSDMAAlhyR4BGYoKWKij1oHW1rpqtXVUO/yp1f6qXVq11db2V8XWWesoah1Vhgg42HtDIBAge+957/f3xzm5uQmBROTeGznP+/XKi3PPPfeeJ8f4nHO+5/t9vmKMQSmllHOEBDsApZRSgaWJXymlHEYTv1JKOYwmfqWUchhN/Eop5TCa+JVSymH8lvhF5DkRKRCRHT7rEkVkqYhk2P8m+Gv/Siml2ufPK/4XgHlt1t0LLDPGDAWW2a+VUkoFkPhzAJeIpADvG2NG26/3AucaY3JFpC+wwhgz3G8BKKWUOkZogPfX2xiTay/nAb2Pt6GI3AzcDBAdHT1pxIgRAQhPKaVOHxs3biwyxiS1XR/oxO9ljDEictzbDWPMAmABQHp6utmwYUPAYlNKqdOBiGS1tz7QvXry7SYe7H8LArx/pZRyvEAn/neB6+zl64B3Arx/pZRyPH9253wVWA0MF5GjIvI94BHgfBHJAObYr5VSSgWQ39r4jTFXH+et2f7ap1JKqY7pyF2llHIYTfxKKeUwmviVUsphNPErpZTDaOJXSimH0cSvlFIOo4lfKaUcRhO/Uko5jCZ+pZRyGE38SinlMJr4lVLKYTTxK6WUw2jiV0oph9HEr5RSDqOJXymlHEYTv1JKOYwmfqWUchhN/Eop5TCa+JVSymE08SullMNo4ldKKYfRxK+UUg6jiV8ppRxGE79SSjmMJn6llHIYTfxKKeUwmviVUsphNPErpZTDaOJXSimH0cSvlFIOo4lfKaUcRhO/Uko5jCZ+pZRyGE38SinlMJr4lVLKYYKS+EXkxyKyU0R2iMirIhIZjDiUUsqJAp74RaQ/8CMg3RgzGnAB3wp0HEop5VTBauoJBbqJSCgQBeQEKQ6llHKcgCd+Y0w28DhwGMgFyo0xS9puJyI3i8gGEdlQWFgY6DCVUuq0FYymngTgEiAV6AdEi8g1bbczxiwwxqQbY9KTkpICHaZSSp22gtHUMwc4aIwpNMY0Am8B04MQh1JKOVIwEv9hYKqIRImIALOB3UGIQymlHCkYbfxrgYXAJmC7HcOCQMehlFJOFRqMnRpjfgn8Mhj7Vkopp9ORu0op5TCa+JVSymE08SullMNo4ldKKYfRxK+UUg6jiV8ppRzmtE78NQ1NHCyqDnYYSinVpZzWif/659dzy8sbaHR7gh2KUkp1Gad14r/prFT25Vdx97+3Ul7bGOxwlFKqSzitE//XRvXhhhkpvLs1hz8vywh2OEop1SUEpWRDIP3y4lHsya1kTWZxsENRSqku4bS+4m82JTWRXbkVVNRpc49SSjki8Z+ZmogxsDGrNNihKKVU0Dki8Q/pFQNAdmltkCNRSqngc0Tij48KB6C0uiHIkSilVPA5IvGHh4YQGxFKSY0mfqWUckTiB0iIDueN9UeY+ehyGpp0QJdSyrkclfirG9wcLqmhTK/8lVIO5pjEnxgV5l12GxPESJRSKrgck/gTosO9y9rUo5RyMsck/sQoTfxKKQUOSvwxkS3VKeo18SulHMwxid832WuZZqWUkzkm8UeEtvyq2tSjlHIyxyT+m2cOZtaIXgA06BW/UsrBHJP4o8JDuXP2UECv+JVSzuaYxA8Q5rJ+XW3jV0o5maMSf7jdzq+9epRSTuaoxN/8gFebepRSTuaoxN98xa8Pd5VSTuaoxO9t49crfqWUgzkq8esVv1JKOS3xu7SNXymlHJX4w1wCaOJXSjlbUBK/iMSLyEIR2SMiu0VkWoD2S3hoCPXa1KOUcrDQjjfxiz8Bi4wxV4hIOBAVqB2Hu0JobNKJWJRSzhXwxC8i3YGZwPUAxpgGIGBzIYaHhtDgdgdqd0op1eUEo6knFSgEnheRzSLydxGJbruRiNwsIhtEZENhYeEp23m4K0Tb+JVSjhaMxB8KTAT+ZoyZAFQD97bdyBizwBiTboxJT0pKOmU7Dw/VxK+UcrZgJP6jwFFjzFr79UKsE0FAhLmERre28SulnKvTbfwiMh1I8f2MMealL7pDY0yeiBwRkeHGmL3AbGDXF/2ekxUe6tIibUopR+tU4heRl4E0YAvQ/GTUAF848dvuAF6xe/RkAjec5Pd8YdbDXU38Sinn6uwVfzow0hhzStpIjDFb7O8MuAhXCHUNbhrdHm/tHqWUcpLOZr4dQB9/BhIoYaHCukMlnP375cEORSmlgqKzV/w9gV0isg6ob15pjPmGX6Lyo+Z6PXkVdZRUN5AYHR7kiJRSKrA6m/gf9GcQgeTbVrVibwGXTUwOWixKKRUMnWrqMcasBA4BYfbyemCTH+Pym5yyWu/yMyszqahrDGI0SikVeJ1K/CLyfaz+9s/Yq/oD//FTTH51pMRK/Pd+fQT7Cip54fNDwQ1IKaUCrLMPd28DZgAVAMaYDKCXv4Lyp9pGqzfqtdMGkZYUw7aj5UGOSCmlAquzib/eLqYGgIiE0rq5/CvjgQvPYGivGKLCQxnZN45dOZr4lVLO0tnEv1JEfg50E5HzgX8D7/kvLP+56ezBLP3JOQCM7BdHTnkdpdUBKw6qlFJB19nEfy9WRc3twC3AB8aY+/0WVYCM7BsHwK7ciiBHopRSgdPZxP+gMeZZY8x8Y8wVwHMi8oo/AwuEccnxhAisO1gS7FCUUipgOpv4B4jIfQB2fZ03gQy/RRUg3aPCGJscz6cZhTyz8gDfWrA62CEppZTfdXYA141YRdXuA84DPjTGPOG/sAJn5tCe/GX5fjYdLgPA4zGEhEhwg1JKKT864RW/iEwUkYnABKx5cq/CutJfaa//yps6uAcen/5JJTX6oFcpdXrr6Ir/D21elwIj7fUGmOWPoAJphP2At1l+RR09YyKCFI1SSvnfCRO/Mea8QAUSLInR4fSKjaCg0qo9l19Rx6h+3YMclVJK+U9nSzZ0F5E/Nk9+LiJ/EJHTJjv6XvXnlVsngPomNx9sz+UUTUGglFJdRmd79TwHVAJX2j8VwPP+CirQRvSJ9S7nV9QB8Ov3d3HrK5vYkFUarLCUUsovOturJ80Yc7nP64dEZIsf4gmKK9OTCXMJr68/wprMYq57bh0r9xUCkFteF+TolFLq1OrsFX+tiJzV/EJEZgC1J9j+K2VIr1junjuCqPBQ1h4s8SZ9gMU78/hge24Qo1NKqVOrs1f8PwBe8mnXLwWu809IwfODc9LYmVPODTNSWLIrnyeW7uO/23JZujOfs4f25PX1R5g3ug/JCVHBDlUppU5aZxN/hTFmnIjEARhjKkQk1Y9xBcW3zxzoXR7SK5bFO/LYerScBreH7/x9LduOlvNJRhEv3TgliFEqpdSX09mmnjfBSvjGmOaKZgv9E1LXUd/k8S431+1fm1lMeW0jH+3K56/L9wcrNKWUOmknvOIXkRHAKKC7iFzm81YcEOnPwLqCyDAXAN8Y14+ocBeXT0pm/tOrWb6ngH9vPMLazBJunJFKqEv494ajXDaxP64QIczV2fOpUkoFXkdNPcOBi4B44GKf9ZXA9/0UU5fx5FXjWbwzj5tnDkZEcHsMsRGhrD9Uws6cCpo8hs1HStlypIxHF+3lLx9nkFNex55fz/OeNBrdHrYdLWfSoIQg/zZKKWXpKPFHAT8DFhhjHFe6MqVnNLeck+Z97QoRxg+M592tOVTWNQFwx782U2xP5JJjd/1ck1nMucOtmSkfW7yXBZ9k8uGdZ3NGm/IQSikVDB21SQzEmm3rURF5UETOFBFHl65MH5ToTfpR4S5v0vd1/fPr+e82qwvo6gPFAN6SEEopFWwnTPzGmN8bY2YBFwBbscozbxKRf4nItSLSOxBBdiUXj+uLCESHu3jnthksvmsmOx6ay+SU1k05jyzaDbRM7p6vA8GUUl1Ep7pzGmMqgbftH0RkJPB14CVgrt+i64IGJ8Ww+1fzqKxrIim2pYrnoB7RrD9Uyi3nDGb1gWJyyqxEX2XfHWSX1WKM4cYX1jO0dyz3zBuB6wvW/a9rdFNR10iv2NP+ubpSyo86qsd/jc/yjOZlY8wuoN4Y46ik3ywyzNUq6QOk9LAGdZ2ZmsjXRvamqKqespoG8iutE0BueS178ipZvreQBZ9kMv/pVXy0K58fvbqZV9cd7lQxuO+/tIEpv12G26OF45RSJ6+jNv6f+Cw/1ea9G09xLF9pY5PjCQ8N4Yy+cQxItE4CS3bm05zP39hwlKufXQPAPfNGsOlwGbf8cyPvbs3hvre2c9bvl/PfbVY10Dq7echXbYObTzOKANiXX9nqPY/H0OAz5kAppU6ko8Qvx1lu77WjzRyWxOZfnE/f7t28JR1+9+FuuncLY+LAeADKahoZkNiNH56bxowhPXB7DPfMG8GvLhlFTUMTv3hnB6+uO8KIXywiI7+SJreHa59bx8KNR7ntX5u8+9qYVcoTS/ex3R5U9sRH+zj/iZW4PYbiqnqq6pt4dd3hdk8gba3NLKaoSh88K+UkHbXxm+Mst/fa8aIjrMM5IKEbYCX6X10yiqW78gH40awhXDC2L2DVBcourWV+ejI9YyIYPyCeb/zlc37+9nYA/rQsg4vG9uOTfYV8YheNu/XcNF5ff4SFG4+y5UgZb2/OZuXd5/LWpmyyy2r5JKOQG55fjyvEGnPwhyX7uP28NK6f0X51jZLqBq5aYN2FrLt/tj47UMohOkr8I0RkG9bVfZq9jP16sF8j+wpLio2gW5iLQT2i+PaUgcwY0pPP9xdx7bQU7zZnD01ixd0tE5yN6d+dvt0jvWWg39+Wyyq7KyhAmEu4a84wskpqvF1FXSHC1qPlZJdZhVJ//pZ10mh+BlBUVc+D7+3i6jMHEhHqOiZO3yajJTvzuWbqoFN0BJRSXVlHiX8c0Bs40mb9ACDPLxGdBkSEp66eQFqvGEJdIaQlxZCWFNPhZyanJPLu1hy+O3WQndTL6BUbwZ68StKSYggPDeEHM9O8if9gUTXX/H0tMRGhjOoXx9qDJQBEhoXwf9+ZSG55Hfe/vYP9BVUMSIziR69uZmxyPHfOHsrCjUd4eU2Wd/87c8pbxePxGN7cdJSLx/XzjkJWSp0eOkr8TwD3GWOyfFfaVTqfoHUZB+VjzsgvPsRhckoC727NYUxyd65MHwDAmxuP8tN/b/WO+h2T3J275w7nva057MmrpKq+iY9+MpP4qHCu/cc6zhuRxE/PH05IiHCgsAqA5z47xKAeUazYW8iKvYUM7x3Lr97bRXWDm/DQECanJLAju8IbR12jm1UHirh74Tb25lXywEUjT8ERUUp1FR0l/t7GmO1tVxpjtotIypfZsYi4gA1AtjHmoi/zXaeL80f2YdHOPM4ZluRdN6y3NS2k7/SQt503hEE9orj9X5uJiQhlSC/rvQ/uPBtjDM2Dq1N6RAPw5qajgHUn0OQ23PPmNqobrAe/DU0eRvfvzvOfHaKhyUNtg5txv1pC7ziru+q6QyUn/ftU1DUy5bcf8eRVE5g3us9Jf49S6tTqqFdP/Ane6/Yl930nsPtLfsdppU/3SF65aSq941oeso7sF8eP5wzj0gn9W207wO45NC2tR6v1vhU1XCFCZFjLf+KBiVEM7xNLVX0To/tbdxDnDU9ibP94Gtweth4t4/ElewHIr7B6+hwsqsbTzriBw8U1LNx4tN3fo6CijoNF1ezJraSu0cNvP9jV6WOglPK/jq74N4jI940xz/quFJGbgI0nu1MRSQYuBH5L67ECqg1XiHDnnKHHrB+b3J2HvjGKb47v386nWrx7+1nUNbp5cVUWV00ewNrMYnbmVPDIZWPp2z2SqPBQ3MYQGRbCfzZns2x3QavPV9Y18fyqQ4QIvL05m5Qe0Tx6xVieXLaPtzZlk1VcTWrPaP6zJYc7Zw9h0qBEfv3f3azaX+SNu3n0cmeV1zayJrOYuaP0LkEpf5ATjRi1a/G8DTTQkujTgXDgUmPMST3gFZGFwMNALPCz9pp6RORm4GaAgQMHTsrKymq7iToJjW4PhZX19ItvfcN212ubWbIrn5oGN0N7xZBRUEVKjygOFdcc8x3XT0/hP1uyKatpbLV+Smoib9wyjW/85TO2HS2nT1wkeRV1uEKE3b+ah8Hg9hiiwk98vXH/29t5Ze1h3r51OhMGajlrpU6WiGw0xqS3Xd9RkbZ8Y8x04CHgkP3zkDFm2pdI+hcBBcaYE94xGGMWGGPSjTHpSUlJJ9pUfQFhrpBjkj7A5ZOSqbHb/a+abD1YnjksidjIliQ9MDGKK9OTeWHVoWOSPlhlKQBvnaK8Cutft8dw7XNrmfX4Si77v1Udxtjkti5GPtlX9EV+NaVUJ3W2SNtyYPkp2ucM4BsicgHWLF5xIvJPY8w1HXxO+dH0tJ70iYukoLKOSyf055lPMpmcksj+gipWHSjmztlDuXBsX3pEh1Pf5MEY+N5ZqVTUNXKwqJoth8t4a3M2qw4UUVRVz7DeMezLr+KcYUkkJ3Rj4caj3qksq+qbiIk4/p+ey2U9p/hodz4/mj2EH/xzI1emDyAmIpTnPj/IPfNGMLiD7rEno6Kukf0FVUzUuwx1muvsZOunjDHmPuA+ABE5F6upR5N+kLlChNtnDWH70XJ6xESw/v45AOzOrWBNZjHXT08hITocgD99a0Krz549NIk5Z9SyYl8h3352LQA3zEhl4cajXDimL1dOHsBlE5N56uMMVuwtZNvRMqan9Wz1He9syaZ3XCRTB/eg1J7jYHduBQWV9Szemc/infneAW578ypbDX47Vf7+6UGeXnGAHQ/NJTz01E2feaCwimv+vpY3bpnmreOkVDDp5LDK65qpg/j9FWNbrbvlnDRev2WaN+kfT7/4brxy05ne1wMTo3jzh9O50m42mjQogSevGg/A5sNlABhjKKluoLbBzb1vbuexxVaPohI78Td5DJsPl3q/M7e8jtkjenGouIaahi/2wLgzMguraHB7KK05dnKdL2PDoRJyy+vYerTslH6vUicrqInfGLNC+/B3bd27hTE5JbFT2/qONejb/di6P/FR4QzvHcs/12SRVVzNW5uymfbwMl5bf5jaRjfbjpZR2+BulXjXHWxJ/NdMHcgldrfWo6W1HcbT0OThSEnrh9ONbg8r9xWSaQ9u89W87fvbclnfwfiFjPxK7nh1c6cK4WUWVQNwuOTYB+VKBYNe8atTxncMQd/u7Q/z+ONV4yiraeTvnx5k+d4C6ps8PPSe1c+/0W2Y9YcV7Muv8p5E1h8qIUTg2WvTeeDCkSTbBfB8E3qj20Oj20NeeZ03ERtjeGn1Ic5/YiXV9U08//lBVh0oYtGOPK57bh2z/rCS2obWSTvL/s5fv7+L+U+v9jY5tbX5cCnPrzrEe1tzWHew4wFuh+zE3/Yk1JVU1DW2O15DnZ4C3savTm/v3j6DlXsL6Rbefn2fUf26Mza5OztyyltdtV8/PYUXVh3yFqk7o28ce/Iq2Z5dTnJCN863S2A0D1xrTqIfbM/l1lc2cf7I3t4qqHPO6MVn+4voExdJXaOH/QVV3pPLreemefe5L7+S3PI6BiRapbTb9lT628oDxEWGcu30FOIiwwBrcNqlPj2T1h4sZuawE/c6O9jFr/iLq+qZ9JuP+NnXhnH7rGPHjKjTj17xq1NqbHI8d8w+cfIY2S+OzYfLKKy0ev/0io3gZ3OH85/bvJO8eUtVAAzq0fJAtGdMOJFhId6Txv12GevmpD9pUAIf7S6grtHjHYPw+YGWbqEbs0oJd1l/9sv2FPCDf27kwj9/xsd78o+Jc8EnmTy+ZB8PvL0DsAaWbTvaupjd2sxjr/iNMd4Z1Twe443jZBP/v9YeZrVPpdZTzbcirHIGTfwq4Eb16+5dfuGGKay5bzYxEaGMHxDP+AHxAPSOi6C55WhM/3jv9iJCckIUR0pryMivpNTnKv2ms1J58cYpnD20dY8h39HIaw+WMGtEL7qFufjzsgzv+mdWZrb6TEJUmHd50Y48KuoaGffQEm56aQMA54/szVXpA9h6tIwmd+vZzx58dyep930AWHMtNzR56BkTQU5ZHY3uLz5T2iMf7uYfn2V2vOFJqm+ymrzCXJoOnEL/S6uAG9XPrjTavzv94rsR4jPpfH+7Db/JY7zTVn5zQr9jPv9pRhGvr7eqhfey5z8e3b87MRGhvPy9M/n+2S2Tz2zMKm31+aG9Y7wT3V88rp+39LUv36JyDW4PL6065H0d5hKevmYSkwYl0Og2vLwmq9XD4BdXW6PMK+oa2Z1rVT29fFJ/3B7DvCc/4Yml+zDGsL+gypt0j6eqvomKuiZvc9EXkZFfyZXPrOaJpft47rODvLrusPe98tpG73GpqLV6SIW6dFK9ruSjXflMf3jZSf2374gmfhVwI/rE8vBlY3jxxinHvHfz2db8PlNTezAuubu9fVyrbe77+hlEhIbw988OkhgdzgVjrFnNmk8oAOcN70WYS0hLsiqURoSG8L92eemBiVFcPz2Fft0j+c0lo70nG9+r/K/ZdYJunJFKdLiLx5fs877nMda4h5Se1nc/9J71MLikuoF/rW1JrrlldezOrUQE7pg1lCG9YjhQWM2flmXw87d3MOePKznn0RUU21NfbjhUwoHCKirrGvnLxxnUNDSRa0+yc6Sk1jvBjq/6Jjd/XLqPgsq6Vk1MAP9ad5h1B0v407IM/rQsg5dWZ3GkpIZhD3zIuIeWcPnfVtHQ5KGs1nqIHRai6aArKa1pIKe8jtCQU39C1oe7KuBEhKunDGz3vXED4jn0yIUAvH7LNJraSXZ9ukdy09mDeWzxXqIjXFwxKZkGt6fVaN7pQ3qy9Zdf481N2fziPzsQgRtmpJDWK4YZaT0IdYXwk/OHERJiNR1tPlzGwB7RlNaUAXDWkJ7cft4Q5qcns/lIKZsPl9E/vhvZZS0JOKVn68FYl/9tVaurs5yyWnbnVpDSI5qYiFBeuGEyO7LLeWXtYe/Vd15FHcv3FnLJ+H5c8fRqAO6eO5zHl+xjX34Vl020uq82uD3klNWSnNCNukaP9+H55/uL+POyDN7flsOUlES2Z5fzrckDGNmvO4t35BHmEhrdhvLaRuoa3Xy0O5+GppbmpsKqeu9DbZcfEow6ebV2D7XjdZT4MjTxqy7rRDN/fXvKQB5bvJdrzhzE6P7d+d2lY47ZJio8lO9OHURsRCjxUWGISKu5DpqbmPrbtYsGJkaxO7eChiYPYa4QfjZ3OAB3zRnGP9dk8ejlY5nw66XMs+8GkmIivN/VOy7imFvynPJadudVeO9EkhOiSE6IYkpqD258YT3XTB3Er9/fxfqDJfTzGffQPJDt3a05bPIZwJZZVM2KfYU8tmgPj88fR12TxztzWmZhNZmF1v5/8c5O72d+NGsIf/54PwD1TZ5jHhLnlddRXmslfm3q6Vqaa2dFaeJXypIQHc6eX88johOlFb454cSlq5ubegYmdmPl3ed627ybnTMsyXvC2PDAHG+dId9xC09eNYGrn13DvV8fwffPHszwBz5kyc58sopr+G6buYwTo8O9PZg+3J7L+qwSuoW7CA+1puncnVvBiD6xpPaM5sMdLbUQP9lXyIasUirqmrj55ZYah2OTu3PJ+P4s31PA6sziVk1C89MH8PQnmd6r/CW7WvdeKqio817xd2Ywmgqc5sQf2c582V+WNuqpr6zIMFer5Huyku0r/kGJ0fTt3o3hPiOQ2+oZE9HqTmTuqN70iYtkWloP3r19Bt87KxVXiNA7LpKV+wqJjQj1lq1oz7S0HmQWVvPymiwuGN2HWSOsE8zQ3rHcbd9xAFw4pi//+OwgW4+UededmWqNqB6SFMP3zkrlnzedySOXjfFO09k7LoLkhG6k2jOxNZs/KdlbPiOvoo4K+4q/ql4Tf1dS29BEtzBXq84Pp4pe8SvHGz8gnjNTE4+ZzawznvluuveB6tjkeO/6fLsk9Q/PS/MO/mrPNVMHsTevkoyCKn71zdGs2m81xSRGhTE4KYaYiFCSYiN46uoJjO7fndfWH2buqD5kFlax4LvpvLj6EHPOaJnfeX76AJITorj62TVMGpSAiDC0dwyVdY14jJXozxvRi6+P7sP/LNzmHdgG+KX+UVfR0OTBY8wJmw+7mpoGt1+aeUATv1IkRIfz+i3TTvrz7d11/PyCM9ibV8kPZqa184kWkWEuHps/zvv6/JG9eeDCM5ifbt0lrLt/NmA9j/jhuWn88NzW33fDjFTaGt0/jtiIUGYOTfLGUlrT4J3Ss6f9bCI6wkVDTcuD3ur6JsprGskqqW51Egs2t8fqrRT6JcYZ/Oa/u9iTW8kbPzj5/86BVtvg9suDXdDEr5Rf3HjWsQm5M1whwk12l1agw9nK2hMbGcbqn88m2k4a/eK7tTv5TmmbEhVV9U3c9fpmlu8tZNMvzifRrsh6uLiGj/fkc+20FL80O3Rk/tOrqGlws+iumSf9HXtyK8koqOx4wyAoqW7wHmtf/rzi1zZ+pU5DMRGhHT7/GJzUuu2/rtHDWrvo3NJdeZRUN1BYWc/Mx5bz4Hu72HK0jFfWZrFoR/ulHWoamvjeC+vZnVvBjuxyXlx1iKKqev7+aSaH25nCszOMMWw6XMaevErveIeTkVdRR1ltY6tR1o1uj7cEeLDsyqlg4q+X8tamo8e8V9PopttJnPg7Q6/4lXKoN26ZRnV9E29vzmZnTgVL7TmXARZuPMo9b25vNVPamsxiHl1kdTU99MiFNDR52HKkjLSkaHrERLD6QDHL9hSQXVZLmCuE7dnlPL54L5X1TRworObhy47tctsR30J+S3bltxr/sXxPAXHdwpg06MQzphljyKuowxjrLifJHun91Mf7eXn1IdbdP6fDchUNTR4yi6qOGUzYbGdOOY1u4y050ln77fLgS3bmc9nE5Fbv1TY0EeWnZxJ6xa+UQ/WMiWBQj2jumjOMWSN6edf3iYtk/SFr/EBVfRPxUWEM6RXDpz5zIBdU1PH6+sNc+cxqZv9xpfckAJBVXOPtPhoWGkKfuEheXXeY9N98xKOL9vC9F9ZT5jPnQn2Tm/0FlWSX1R5TunpnToV3+fP9RazNLKa8tpF739zGDS+s5+pn13D531bxaUYhYDVL+Y59ACirafTG43uFv3RXPqU1jezNO3ET0Htbc3jovZ3Me/JT751LTlltq+6v97+9g3sWbjvh9/iOqm7WfE9W107pDn24q5Tyq2ifK/vH54/jztc2U2wnySkpifSOi+TlNVnebT7bX+RtFiqraeTB93Z6y1XUNrrZm1/JZRP688BFI3lt/WEeXbSXoqp6/m/FAQDuen0L0eGhrNhbwMRBCaw6UEzv2Agiwlws+8k5hIQI72zJ5s7XtgAwY0gPlu7K5/1tuYzoE8ve/ErCXEJDk4eNWaX87oM9fHhnEj95Yws7cspZ+uNzvNNc5tk9rAAKKut4afUhymtb6ij94p0d3HruEG/pb7BqGTW6PZRWN3DHq5u967dnlxMd4WL6Ix/znTMH8ttLx9DQ5GFXbgUY60F0eyOgj5TUcNFTn/Hc9ZNb3aE0nwDbG0Phz4e7esWvlCImoiXBTElN5PVbpvGvm85EBGYM6cm4Nk0Yr647zObDZd5k2Zz0509qaa6YMDCexOjwY2ZwG947lhV7C/nv9lyqG9x8mlGE22PIKa/jYFE1K/dZV++fZlh3GFemJzN+QDz19lX7nrxKhvWK5cM7Wx72hoYIGfmVbMgqpa7R06qbqm/i/yyjiFfWHm5Vgnrz4TJue2UTO7LLMcYqb3HjC+tJ/81Hx5Th3nKklNc3WMUBF+3IY01mMec+tpyGJg8Nbg9HS9t/lvHOlmzKaxvZ1KZgYEm19YC9tvHYqq16xa+U8qtwl5VgLhnfj/DQEIb0imFIrxjev+MshvWOZX9By1SV3z87lWc/PQhY9Y9G9o1jX34lV04ewPjkeP690XpQmWbXTkoflMCjl48lrVcMH27P5eJx/bjkr58THe5i3ui+vLnpKPFRYd55Ep74aB+/X7SH/QVVnDWkJ49eMY53tmS3indUvzjSkqKZkpLIukMlbM8u5763thPuCuH6GSks+CSTxTvzmDuqDwU+iX/FXuuk8sYt0yiraeAPS/axN7+SmMhQ/rp8PyP6xPHERy0F+R74z45W+23+vcGqvnrPm9vIKW/5/gOFVQzqEc22o2V8uCOPc4YlMXVwD/673RqBfai4dVmP5mlGC31ibFbT0HRSvbo6QxO/UoqpgxP5zTdHc3mbB4zNcycM6dVSAO+uOcM4UlLLsj35zByW1GrSHIDLJvTnrc3Z3s+IiHf08qRBCRhjGNorhvSUBK6dlsLu3AqevmYSsZGhvLwmiz8ubUm8zd/d9qHqyH5xiAhv/GAa723N4Y5XN7Mhq5QnrxrPhWP78llGET9+fQtPXzOJ19Yfobn1ZW9+pT2PtDW4bfyAeGoa3Dz3+UFeWp3VqkQGtBRK8/XT84cRHxXGL97ZSVZxDRGhId67kQMF1SQnVHLF31bT4PbwtxUH+Ob4ft5mpcU784mNDON/5g4nJES8zxzyKupocntajVWobdR+/EopPwp1hXBNm5pCvnx7vURHhPL0dycdd9uHLx/D9TNS6BUX2e77IsK7t59FqEsIc4XwwZ1ne9+7espAnll5gGq7d9Gw3tbJY3BSNOcMS6JbmItFO/NaTeZz1pCezB7Ri2+fOZDZ9ijmF26YzAV//pTrnl9HmCuEhy8bwyMf7qG0ppHR/eO8XV2bY7xobD9eWt3yDANg3qg+LNrZciK4e+5wbpyRSrdwF6XVDby0OotvTujP988eTHV9E+c/sZJ9+ZWEh4bQ4Paw9MczeXJZBv/ZkgNYJTbWHizh6ZUHuHRCf1YdKGKx/f0eA/mV9d6CgdY80kZ79Silguupqyfwy4tHdrhdRKirw5G/3cJd7XahTIqN4LN7ZnHLTGsQW6o950GYK4QXb5zCI5eP4a45Q0lPaXlAmhAdzj+un+xN+mAl9FvPHYIxcMvMwVw1eSAue76Bs4ceO0fy5JQEHrtiLIvvmsnUwdYziR+1mUI0pUe09wo8ITqcpT85h9vOG0J4aAgJ0eGMTY5n4+FS9hdUERsRypBeMdw7bwQAIrSK+aH3dvLQe7uob/LQM8YavPWp/WzDGOOdJEev+JVSQXXxuH4db3QKJESH87O5w5ma1oMpqa0fDMdHhXPXnGGd+p5rpw2iX3w3zrML3z119QTKaxv42sg+x2wrIt4yGReM6Ut1vZsz+sby3u1n0eTxcNsrmxib3P2Yz/k6MzWRj/cUECJCWq8YRIQBiVE8dfUEBiZGsTqzpST2qgPFiIAxMD2tJ7tzK3hjwxGumjyA655fzyf2SUATv1LKMcJcIZw3vFfHG55AqCuk1RSanS3Cd+20FK6dlgLAGDvZr7pvdoefmzrY+v79BVVc4dO7qfmEObxPLN27hbH1SBkfbM/ljllD+e0Hu8kuq+WKSck8/OEeFu/M9yZ9gG7a1KOUUl3XqH5WcTxoaaLyFRnm4uopA/nlxaNY9tNzucSeS3pEn1jm2pP73PrKRsJcwvXTUwCrjIY/6BW/UkqdAqGuEN68dTr/s3Ab5w4/9jlCs27hLm8Tzkc/mUlyQhSRYS4SosIorWnkwtF9uf/CM0jtGc2lHUwidLKkvWHEXU16errZsGFDsMNQSim/eW9rDqsOFPG/F406ZW37IrLRGJPedr1e8SulVBdw8bh+AXuArm38SinlMJr4lVLKYTTxK6WUw2jiV0oph9HEr5RSDqOJXymlHCbgiV9EBojIchHZJSI7ReTOQMeglFJOFox+/E3AT40xm0QkFtgoIkuNMbs6+qBSSqkvL+BX/MaYXGPMJnu5EtgN+GdcslJKqWMEtY1fRFKACcDadt67WUQ2iMiGwsLCYz6rlFLq5AQt8YtIDPAmcJcxpqLt+8aYBcaYdGNMelLS8QseKaWU+mKCkvhFJAwr6b9ijHkrGDEopZRTBaNXjwD/AHYbY/4Y6P0rpZTTBeOKfwbwXWCWiGyxfy4IQhxKKeVIAe/OaYz5DJBA71cppZRFR+4qpZTDaOJXSimH0cSvlFIOo4lfKaUcRhO/Uko5jCZ+pZRyGE38SinlMJr4lVLKYTTxK6WUw2jiV0oph9HEr5RSDqOJXymlHEYTv1JKOYwmfqWUchhN/Eop5TCa+JVSymE08SullMNo4ldKKYfRxK+UUg6jiV8ppRxGE79SSjmMJn6llHIYTfxKKeUwmviVUsphNPErpZTDaOJXSimH0cSvlFIOo4lfKaUcRhO/Uko5jCZ+pZRyGE38SinlMJr4lVLKYTTxK6WUw2jiV0oph9HEr5RSDhOUxC8i80Rkr4jsF5F7gxGDUko5VcATv4i4gL8CXwdGAleLyMhAx6GUUk4VjCv+KcB+Y0ymMaYBeA24JAhxKKWUI4UGYZ/9gSM+r48CZ7bdSERuBm62X1aJyN6T3F9PoOgkPxtoGuup91WJEzRWf/mqxOqPOAe1tzIYib9TjDELgAVf9ntEZIMxJv0UhOR3Guup91WJEzRWf/mqxBrIOIPR1JMNDPB5nWyvU0opFQDBSPzrgaEikioi4cC3gHeDEIdSSjlSwJt6jDFNInI7sBhwAc8ZY3b6cZdfurkogDTWU++rEidorP7yVYk1YHGKMSZQ+1JKKdUF6MhdpZRyGE38SinlMKd14u/KpSFE5JCIbBeRLSKywV6XKCJLRSTD/jchSLE9JyIFIrLDZ127sYnlz/Yx3iYiE7tArA+KSLZ9bLeIyAU+791nx7pXROYGMM4BIrJcRHaJyE4RudNe3+WO6wli7YrHNVJE1onIVjvWh+z1qSKy1o7pdbsjCSISYb/eb7+f0gVifUFEDvoc1/H2ev/9DRhjTssfrAfHB4DBQDiwFRgZ7Lh84jsE9Gyz7lHgXnv5XuD3QYptJjAR2NFRbMAFwIeAAFOBtV0g1geBn7Wz7Uj77yACSLX/PlwBirMvMNFejgX22fF0ueN6gli74nEVIMZeDgPW2sfrDeBb9vqngR/ay7cCT9vL3wJeD+BxPV6sLwBXtLO93/4GTucr/q9iaYhLgBft5ReBbwYjCGPMJ0BJm9XHi+0S4CVjWQPEi0jfgATKcWM9nkuA14wx9caYg8B+rL8TvzPG5BpjNtnLlcBurFHsXe64niDW4wnmcTXGmCr7ZZj9Y4BZwEJ7fdvj2ny8FwKzRUSCHOvx+O1v4HRO/O2VhjjRH2+gGWCJiGwUqzwFQG9jTK69nAf0Dk5o7TpebF31ON9u3x4/59Nk1iVitZsXJmBd8XXp49omVuiCx1VEXCKyBSgAlmLdcZQZY5raiccbq/1+OdAjWLEaY5qP62/t4/qEiES0jdV2yo7r6Zz4u7qzjDETsaqU3iYiM33fNNa9Xpfsa9uVY7P9DUgDxgO5wB+CGo0PEYkB3gTuMsZU+L7X1Y5rO7F2yeNqjHEbY8ZjVQGYAowIbkTH1zZWERkN3IcV82QgEbjH33Gczom/S5eGMMZk2/8WAG9j/cHmN9/K2f8WBC/CYxwvti53nI0x+fb/YB7gWVqaHYIaq4iEYSXSV4wxb9mru+RxbS/WrnpcmxljyoDlwDSsZpHmAaq+8Xhjtd/vDhQHNtJWsc6zm9aMMaYeeJ4AHNfTOfF32dIQIhItIrHNy8DXgB1Y8V1nb3Yd8E5wImzX8WJ7F7jW7oEwFSj3aboIijbtoJdiHVuwYv2W3bMjFRgKrAtQTAL8A9htjPmjz1td7rgeL9YuelyTRCTeXu4GnI/1TGI5cIW9Wdvj2ny8rwA+tu+0ghXrHp8Tv2A9i/A9rv75G/DXE+yu8IP1VHwfVpvf/cGOxyeuwVi9ILYCO5tjw2prXAZkAB8BiUGK71WsW/lGrHbF7x0vNqweB3+1j/F2IL0LxPqyHcs2+3+evj7b32/Huhf4egDjPAurGWcbsMX+uaArHtcTxNoVj+tYYLMd0w7gf+31g7FOPvuBfwMR9vpI+/V++/3BXSDWj+3jugP4Jy09f/z2N6AlG5RSymFO56YepZRS7dDEr5RSDqOJXymlHEYTv1JKOYwmfqWUchhN/MpxRMRtV0HcKiKbRGR6B9vHi8itnfjeFSLS6cmyReRVe5zJXSJydWc/p9SXpYlfOVGtMWa8MWYc1nD5hzvYPh6rquOplmKsombnAJ/44fuVapcmfuV0cUApWLVpRGSZfRewXUSaq7k+AqTZdwmP2dveY2+zVUQe8fm++XbN9X0icnZ7OxSRV0RkFzDCLtj1NeC/InKTv35JpXwFfLJ1pbqAbnbCjcSqPT/LXl8HXGqMqRCRnsAaEXkXq07+aGMV10JEvo5VMvdMY0yNiCT6fHeoMWaKWJOU/BKY03bnxpjviMh8YCBWaeDHjTHz/fGLKtUeTfzKiWp9kvg04CW7SqIAv7MrpXqwSuC2Vxp7DvC8MaYGwBjjOx9Ac/G1jUDKCWKYiFWqYSxW6Q6lAkYTv3I0Y8xq++o+CaseTRIwyRjTKCKHsO4Kvoh6+1837fz/Zd8J/A5rpqqL7P1Vi8hsY8x5J/dbKPXFaBu/cjQRGYE1TWcxVoneAjvpnwcMsjerxJqCsNlS4AYRibK/w7ep54SMMR8Ak7CmihyDVaRvgiZ9FUh6xa+cqLmNH6zmneuMMW4ReQV4T0S2AxuAPQDGmGIR+VysCd0/NMbcLdaE2BtEpAH4APj5F9j/BGCrXS48zLSZkEUpf9PqnEop5TDa1KOUUg6jiV8ppRxGE79SSjmMJn6llHIYTfxKKeUwmviVUsphNPErpZTD/D8NdoAjowtnfwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "#Plot result of Loss \n",
        "plt.plot(batch_loss.logs)\n",
        "plt.ylim([0, 10])\n",
        "plt.xlabel('Batch #')\n",
        "plt.ylabel('CE/token')"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "e78d873b",
      "metadata": {
        "id": "e78d873b"
      },
      "source": [
        "### Translate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "303aa3eb",
      "metadata": {
        "id": "303aa3eb"
      },
      "outputs": [],
      "source": [
        "class Translator(tf.Module):\n",
        "\n",
        "  def __init__(self, encoder, decoder, input_text_processor,\n",
        "               output_text_processor):\n",
        "    self.encoder = encoder\n",
        "    self.decoder = decoder\n",
        "    self.input_text_processor = input_text_processor\n",
        "    self.output_text_processor = output_text_processor\n",
        "\n",
        "    self.output_token_string_from_index = (\n",
        "        tf.keras.layers.experimental.preprocessing.StringLookup(\n",
        "            vocabulary=output_text_processor.get_vocabulary(),\n",
        "            mask_token='',\n",
        "            invert=True))\n",
        "\n",
        "    # The output should never generate padding, unknown, or start.\n",
        "    index_from_string = tf.keras.layers.experimental.preprocessing.StringLookup(\n",
        "        vocabulary=output_text_processor.get_vocabulary(), mask_token='')\n",
        "    token_mask_ids = index_from_string(['', '[UNK]', '[START]']).numpy()\n",
        "\n",
        "    token_mask = np.zeros([index_from_string.vocabulary_size()], dtype=np.bool)\n",
        "    token_mask[np.array(token_mask_ids)] = True\n",
        "    self.token_mask = token_mask\n",
        "\n",
        "    self.start_token = index_from_string(tf.constant('[START]'))\n",
        "    self.end_token = index_from_string(tf.constant('[END]'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ed97a050",
      "metadata": {
        "id": "ed97a050"
      },
      "outputs": [],
      "source": [
        "translator = Translator(\n",
        "    encoder=train_translator.encoder,\n",
        "    decoder=train_translator.decoder,\n",
        "    input_text_processor=input_text_processor,\n",
        "    output_text_processor=output_text_processor,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "8a58cc45",
      "metadata": {
        "id": "8a58cc45"
      },
      "source": [
        "**Convert token IDs to text**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3839ef9f",
      "metadata": {
        "id": "3839ef9f"
      },
      "outputs": [],
      "source": [
        "def tokens_to_text(self, result_tokens):\n",
        "  shape_checker = ShapeChecker()\n",
        "  shape_checker(result_tokens, ('batch', 't'))\n",
        "  result_text_tokens = self.output_token_string_from_index(result_tokens)\n",
        "  shape_checker(result_text_tokens, ('batch', 't'))\n",
        "\n",
        "  result_text = tf.strings.reduce_join(result_text_tokens,\n",
        "                                       axis=1, separator=' ')\n",
        "  shape_checker(result_text, ('batch'))\n",
        "\n",
        "  result_text = tf.strings.strip(result_text)\n",
        "  shape_checker(result_text, ('batch',))\n",
        "  return result_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2b851efe",
      "metadata": {
        "id": "2b851efe"
      },
      "outputs": [],
      "source": [
        "Translator.tokens_to_text = tokens_to_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0b8b741f",
      "metadata": {
        "id": "0b8b741f",
        "outputId": "a6e26c1a-4a57-41b5-fe53-b5736e0b6afa"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([b'impress chance', b'hungry down', b'fallen cry', b'no leave',\n",
              "       b'drives shut'], dtype=object)"
            ]
          },
          "execution_count": 61,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Input ome random token IDs and see what it generates\n",
        "example_output_tokens = tf.random.uniform(\n",
        "    shape=[5, 2], minval=0, dtype=tf.int64,\n",
        "    maxval=output_text_processor.vocabulary_size())\n",
        "translator.tokens_to_text(example_output_tokens).numpy()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "04d80536",
      "metadata": {
        "id": "04d80536"
      },
      "source": [
        "**Sample from the decoder's predictions**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f734b981",
      "metadata": {
        "id": "f734b981"
      },
      "outputs": [],
      "source": [
        "def sample(self, logits, temperature):\n",
        "  shape_checker = ShapeChecker()\n",
        "  # 't' is usually 1 here.\n",
        "  shape_checker(logits, ('batch', 't', 'vocab'))\n",
        "  shape_checker(self.token_mask, ('vocab',))\n",
        "\n",
        "  token_mask = self.token_mask[tf.newaxis, tf.newaxis, :]\n",
        "  shape_checker(token_mask, ('batch', 't', 'vocab'), broadcast=True)\n",
        "\n",
        "  # Set the logits for all masked tokens to -inf, so they are never chosen.\n",
        "  logits = tf.where(self.token_mask, -np.inf, logits)\n",
        "\n",
        "  if temperature == 0.0:\n",
        "    new_tokens = tf.argmax(logits, axis=-1)\n",
        "  else: \n",
        "    logits = tf.squeeze(logits, axis=1)\n",
        "    new_tokens = tf.random.categorical(logits/temperature,\n",
        "                                        num_samples=1)\n",
        "  \n",
        "  shape_checker(new_tokens, ('batch', 't'))\n",
        "\n",
        "  return new_tokens\n",
        "\n",
        "Translator.sample = sample"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "71f28f4c",
      "metadata": {
        "id": "71f28f4c",
        "outputId": "07d7e95d-d786-4369-f25d-46bad15f6814"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(5, 1), dtype=int64, numpy=\n",
              "array([[2607],\n",
              "       [ 318],\n",
              "       [3215],\n",
              "       [2280],\n",
              "       [ 950]], dtype=int64)>"
            ]
          },
          "execution_count": 63,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Test run this function on some random inputs\n",
        "example_logits = tf.random.normal([5, 1, output_text_processor.vocabulary_size()])\n",
        "example_output_tokens = translator.sample(example_logits, temperature=1.0)\n",
        "example_output_tokens"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "f2c012ae",
      "metadata": {
        "id": "f2c012ae"
      },
      "source": [
        "**Implement the translation loop**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b75e12da",
      "metadata": {
        "id": "b75e12da"
      },
      "outputs": [],
      "source": [
        "def translate_unrolled(self,\n",
        "                       input_text, *,\n",
        "                       max_length=50,\n",
        "                       return_attention=True,\n",
        "                       temperature=1.0):\n",
        "  batch_size = tf.shape(input_text)[0]\n",
        "  input_tokens = self.input_text_processor(input_text)\n",
        "  enc_output, enc_state = self.encoder(input_tokens)\n",
        "\n",
        "  dec_state = enc_state\n",
        "  new_tokens = tf.fill([batch_size, 1], self.start_token)\n",
        "\n",
        "  result_tokens = []\n",
        "  attention = []\n",
        "  done = tf.zeros([batch_size, 1], dtype=tf.bool)\n",
        "\n",
        "  for _ in range(max_length):\n",
        "    dec_input = DecoderInput(new_tokens=new_tokens,\n",
        "                             enc_output=enc_output,\n",
        "                             mask=(input_tokens!=0))\n",
        "\n",
        "    dec_result, dec_state = self.decoder(dec_input, state=dec_state)\n",
        "\n",
        "    attention.append(dec_result.attention_weights)\n",
        "\n",
        "    new_tokens = self.sample(dec_result.logits, temperature)\n",
        "\n",
        "    # If a sequence produces an `end_token`, set it `done`\n",
        "    done = done | (new_tokens == self.end_token)\n",
        "    # Once a sequence is done it only produces 0-padding.\n",
        "    new_tokens = tf.where(done, tf.constant(0, dtype=tf.int64), new_tokens)\n",
        "\n",
        "    # Collect the generated tokens\n",
        "    result_tokens.append(new_tokens)\n",
        "\n",
        "    if tf.executing_eagerly() and tf.reduce_all(done):\n",
        "      break\n",
        "\n",
        "  # Convert the list of generates token ids to a list of strings.\n",
        "  result_tokens = tf.concat(result_tokens, axis=-1)\n",
        "  result_text = self.tokens_to_text(result_tokens)\n",
        "\n",
        "  if return_attention:\n",
        "    attention_stack = tf.concat(attention, axis=1)\n",
        "    return {'text': result_text, 'attention': attention_stack}\n",
        "  else:\n",
        "    return {'text': result_text}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "109be655",
      "metadata": {
        "id": "109be655"
      },
      "outputs": [],
      "source": [
        "Translator.translate = translate_unrolled"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "7f720297",
      "metadata": {
        "id": "7f720297",
        "outputId": "87f8d175-91b8-4f9d-cba6-2b288fc7587d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "you have any quickly for common .\n",
            "tom grabbed a lot and web rope conditions to play lying .\n",
            "\n",
            "Wall time: 2.34 s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "input_text = tf.constant([\n",
        "    'Bạn không thể tin tưởng vào máy dịch được.', # \"You can't trust computer translation.\"\n",
        "    'Tom đánh piano giỏi hơn Mary.' # \"Tom plays the piano better than Mary does.\"\n",
        "])\n",
        "\n",
        "result = translator.translate(\n",
        "    input_text = input_text)\n",
        "\n",
        "print(result['text'][0].numpy().decode())\n",
        "print(result['text'][1].numpy().decode())\n",
        "print()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9d6b09a4",
      "metadata": {
        "id": "9d6b09a4"
      },
      "outputs": [],
      "source": [
        "@tf.function(input_signature=[tf.TensorSpec(dtype=tf.string, shape=[None])])\n",
        "def tf_translate(self, input_text):\n",
        "  return self.translate(input_text)\n",
        "\n",
        "Translator.tf_translate = tf_translate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "544a365f",
      "metadata": {
        "id": "544a365f",
        "outputId": "f0095f74-0c1c-45bb-fd66-c1d047b8195b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Wall time: 1min 30s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "result = translator.tf_translate(\n",
        "    input_text = input_text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9e8c6096",
      "metadata": {
        "id": "9e8c6096",
        "outputId": "a0a4b8d2-7925-4646-8682-7439099fc85e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "you dont know logged almost you eaten fallen .\n",
            "tom changed the idea tom is here .\n",
            "\n",
            "Wall time: 434 ms\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "result = translator.tf_translate(\n",
        "    input_text = input_text)\n",
        "\n",
        "print(result['text'][0].numpy().decode())\n",
        "print(result['text'][1].numpy().decode())\n",
        "print()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "39438a96",
      "metadata": {
        "id": "39438a96"
      },
      "source": [
        "### Visualize the process"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c9e9135f",
      "metadata": {
        "id": "c9e9135f",
        "outputId": "89e84c17-2857-4143-b69f-24d316f33b5e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.99999994 1.0000001  0.99999994 1.         0.9999999  0.99999994\n",
            " 0.99999994 1.         1.         1.0000001  1.         0.9999999\n",
            " 1.         0.99999994 0.99999994 1.         1.         0.99999994\n",
            " 1.         1.0000001  1.         1.         1.         1.\n",
            " 0.99999994 0.99999994 1.         1.         1.         0.99999994\n",
            " 1.         1.         1.         1.0000001  1.0000001  1.\n",
            " 1.         1.         1.         0.99999994 0.99999994 1.\n",
            " 0.99999994 0.9999999  1.         1.         1.         0.99999994\n",
            " 0.99999994 0.99999994]\n"
          ]
        }
      ],
      "source": [
        "a = result['attention'][0]\n",
        "\n",
        "print(np.sum(a, axis=-1))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0d71c225",
      "metadata": {
        "id": "0d71c225",
        "outputId": "59a4e38c-7be5-4ce3-e083-548365391307"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOeElEQVR4nO3df6zdd13H8eeL1qoMAppeCbYdt9GCaRAdXMuUBAlsSZeZlgSELoGAGTYmVKYjaqdmmvrPAEMksTHUOQUF6pxEr+5qJYAxmoz07keAtlaupay3gruMCUYjXcPbP+4ZHC63Pd/2fm9P72fPR9L0fL/nk3Pe37Z59tvv+dFUFZKkte8Z4x5AktQPgy5JjTDoktQIgy5JjTDoktSI9eN64o0bN9bk5OS4nl6S1qQHH3zwy1U1sdx9Ywv65OQks7Oz43p6SVqTknzhQvd5yUWSGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGjG2T4rqWyb339/7Y56+6+beH1PS1c0zdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqRKegJ9mZ5GSSuST7L7DmDUmOJzmW5MP9jilJGmXk1+cmWQccBG4E5oGjSaar6vjQmm3AHcArquqJJD+wWgNLkpbX5Qx9BzBXVaeq6hxwGNi9ZM3PAwer6gmAqnqs3zElSaN0Cfom4MzQ9vxg37AXAi9M8i9JHkiyc7kHSrI3yWyS2YWFhcubWJK0rL5eFF0PbANeBdwC/GGS5y5dVFWHqmqqqqYmJiZ6empJEnQL+llgy9D25sG+YfPAdFU9WVWfB/6NxcBLkq6QLkE/CmxLsjXJBmAPML1kzV+xeHZOko0sXoI51d+YkqRRRga9qs4D+4AjwAng3qo6luRAkl2DZUeAx5McBz4J/EpVPb5aQ0uSvtPIty0CVNUMMLNk351Dtwu4ffBDkjQGflJUkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEZ2CnmRnkpNJ5pLsX+b+tyZZSPLI4Mfb+h9VknQx60ctSLIOOAjcCMwDR5NMV9XxJUv/vKr2rcKMkqQOupyh7wDmqupUVZ0DDgO7V3csSdKl6hL0TcCZoe35wb6lXpfk00nuS7JluQdKsjfJbJLZhYWFyxhXknQhfb0o+jfAZFW9BPgY8IHlFlXVoaqaqqqpiYmJnp5akgTdgn4WGD7j3jzY901V9XhVfX2weTfwsn7GkyR11SXoR4FtSbYm2QDsAaaHFyR5/tDmLuBEfyNKkroY+S6XqjqfZB9wBFgH3FNVx5IcAGarahp4R5JdwHngK8BbV3FmSdIyRgYdoKpmgJkl++4cun0HcEe/o0mSLoWfFJWkRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRnQKepKdSU4mmUuy/yLrXpekkkz1N6IkqYuRQU+yDjgI3ARsB25Jsn2Zdc8GbgM+1feQkqTRupyh7wDmqupUVZ0DDgO7l1n3O8C7gP/rcT5JUkddgr4JODO0PT/Y901JXgpsqar7L/ZASfYmmU0yu7CwcMnDSpIubMUviiZ5BvBe4J2j1lbVoaqaqqqpiYmJlT61JGlIl6CfBbYMbW8e7HvKs4EXA/+Y5DRwPTDtC6OSdGV1CfpRYFuSrUk2AHuA6afurKqvVtXGqpqsqkngAWBXVc2uysSSpGWNDHpVnQf2AUeAE8C9VXUsyYEku1Z7QElSN+u7LKqqGWBmyb47L7D2VSsfS5J0qfykqCQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1olPQk+xMcjLJXJL9y9z/C0k+k+SRJP+cZHv/o0qSLmZk0JOsAw4CNwHbgVuWCfaHq+pHq+rHgXcD7+17UEnSxXU5Q98BzFXVqao6BxwGdg8vqKqvDW1eA1R/I0qSuljfYc0m4MzQ9jzw8qWLkrwduB3YALx6uQdKshfYC3Dttdde6qySpIvo7UXRqjpYVT8E/BrwmxdYc6iqpqpqamJioq+nliTRLehngS1D25sH+y7kMPDaFcwkSboMXYJ+FNiWZGuSDcAeYHp4QZJtQ5s3A5/rb0RJUhcjr6FX1fkk+4AjwDrgnqo6luQAMFtV08C+JDcATwJPAG9ZzaElSd+py4uiVNUMMLNk351Dt2/reS5J0iXyk6KS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmN6BT0JDuTnEwyl2T/MvffnuR4kk8n+XiSF/Q/qiTpYkYGPck64CBwE7AduCXJ9iXLHgamquolwH3Au/seVJJ0cV3O0HcAc1V1qqrOAYeB3cMLquqTVfW/g80HgM39jilJGqVL0DcBZ4a25wf7LuRW4O+WuyPJ3iSzSWYXFha6TylJGqnXF0WTvAmYAt6z3P1VdaiqpqpqamJios+nlqSnvfUd1pwFtgxtbx7s+zZJbgB+A/jpqvp6P+NJkrrqcoZ+FNiWZGuSDcAeYHp4QZLrgPcDu6rqsf7HlCSNMjLoVXUe2AccAU4A91bVsSQHkuwaLHsP8CzgL5I8kmT6Ag8nSVolXS65UFUzwMySfXcO3b6h57kkSZfIT4pKUiMMuiQ1wqBLUiMMuiQ1wqBLUiM6vcvl6Wpy//29P+bpu27u/TElCTxDl6RmGHRJaoRBl6RGGHRJaoRBl6RGGHRJaoRBl6RGGHRJaoRBl6RGGHRJaoRBl6RGGHRJaoRBl6RG+G2LTyN9f3uk3xwpXV08Q5ekRhh0SWqEQZekRnQKepKdSU4mmUuyf5n7X5nkoSTnk7y+/zElSaOMDHqSdcBB4CZgO3BLku1Llj0KvBX4cN8DSpK66fIulx3AXFWdAkhyGNgNHH9qQVWdHtz3jVWYUZLUQZdLLpuAM0Pb84N9lyzJ3iSzSWYXFhYu5yEkSRdwRV8UrapDVTVVVVMTExNX8qklqXldgn4W2DK0vXmwT5J0FekS9KPAtiRbk2wA9gDTqzuWJOlSjQx6VZ0H9gFHgBPAvVV1LMmBJLsAkvxEknngZ4H3Jzm2mkNLkr5Tp+9yqaoZYGbJvjuHbh9l8VKMJGlM/KSoJDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIzr9J9FSV5P77+/9MU/fdXPvjym1yDN0SWqEQZekRnS65JJkJ/A+YB1wd1XdteT+7wY+CLwMeBx4Y1Wd7ndU6Vu8tHN5+v51ezr8mq0lI4OeZB1wELgRmAeOJpmuquNDy24FnqiqH06yB3gX8MbVGFjS1e1K/WXb2vP0ocsZ+g5grqpOASQ5DOwGhoO+G/jtwe37gN9PkqqqHmf9prX0C6y1ba1Gwz/PT08Z1dwkrwd2VtXbBttvBl5eVfuG1nx2sGZ+sP3vgzVfXvJYe4G9g80XASf7OpCL2Ah8eeSqtaGlY4G2jqelYwGP52r2gqqaWO6OK/q2xao6BBy6ks+ZZLaqpq7kc66Wlo4F2jqelo4FPJ61qsu7XM4CW4a2Nw/2LbsmyXrgOSy+OCpJukK6BP0osC3J1iQbgD3A9JI108BbBrdfD3xita6fS5KWN/KSS1WdT7IPOMLi2xbvqapjSQ4As1U1DfwR8KdJ5oCvsBj9q8UVvcSzylo6FmjreFo6FvB41qSRL4pKktYGPykqSY0w6JLUiGaDnmRnkpNJ5pLsH/c8K5FkS5JPJjme5FiS28Y900olWZfk4SR/O+5ZVirJc5Pcl+Rfk5xI8pPjnulyJfnlwZ+xzyb5SJLvGfdMlyLJPUkeG3w25ql935/kY0k+N/j5+8Y542pqMuhDX1dwE7AduCXJ9vFOtSLngXdW1XbgeuDta/x4AG4DTox7iJ68D/j7qvoR4MdYo8eVZBPwDmCqql7M4psgrqY3OHTxJ8DOJfv2Ax+vqm3AxwfbTWoy6Ax9XUFVnQOe+rqCNamqvlhVDw1u/zeLwdg03qkuX5LNwM3A3eOeZaWSPAd4JYvv9KKqzlXVf411qJVZD3zv4PMkzwT+Y8zzXJKq+icW32k3bDfwgcHtDwCvvZIzXUmtBn0TcGZoe541HMBhSSaB64BPjXmUlfg94FeBb4x5jj5sBRaAPx5cQro7yTXjHupyVNVZ4HeBR4EvAl+tqn8Y71S9eF5VfXFw+0vA88Y5zGpqNehNSvIs4C+BX6qqr417nsuR5GeAx6rqwXHP0pP1wEuBP6iq64D/YY3+k35wbXk3i39J/SBwTZI3jXeqfg0+8Njse7VbDXqXrytYU5J8F4sx/1BVfXTc86zAK4BdSU6zeCns1Un+bLwjrcg8MF9VT/2L6T4WA78W3QB8vqoWqupJ4KPAT415pj78Z5LnAwx+fmzM86yaVoPe5esK1owkYfEa7Ymqeu+451mJqrqjqjZX1SSLvy+fqKo1exZYVV8CziR50WDXa/j2r5ZeSx4Frk/yzMGfudewRl/gXWL4q0neAvz1GGdZVU3+J9EX+rqCMY+1Eq8A3gx8Jskjg32/XlUz4xtJQ34R+NDg5OEU8HNjnueyVNWnktwHPMTiO6seZo19ZD7JR4BXARuTzAO/BdwF3JvkVuALwBvGN+Hq8qP/ktSIVi+5SNLTjkGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqxP8D2AGsaMIFZ5EAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "_ = plt.bar(range(len(a[0, :])), a[0, :])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "aeed0630",
      "metadata": {
        "id": "aeed0630",
        "outputId": "066c7b6b-7cd6-4b9f-face-7bdb658673e8"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x275a2eab6d0>"
            ]
          },
          "execution_count": 77,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAFYAAAD6CAYAAAA7iLBiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAKyklEQVR4nO2dW4xdVRnHf/8zl16pZeglDQMFtYH0RUywYnwDmhQ01ohBiDHEkPCiCUYTqfpk4gO+eEnwwUYRTAxIxMSGoAYKBEgMN1GugZYJl5ZCK5eWXujM9Hw+nD3tWWfvc86ec+bbc/t+ycmctc86e+35zZq111p7f2vLzAhmntpsH8BCJcQ6EWKdCLFOhFgnQqwTfYmVtE3SK5L2StoxUwe1EFCv/VhJA8CrwFZgH/AUcL2ZvdTuO8NaYktZcTpdX70il+fi8w8l6VfHzsmXffxkkjarpxkq6pp/zDHG7aSKPhvsY79bgL1mNgYg6W5gO9BW7FJW8PnalafTx6/Yksvz2G2/TdJbv/HtXJ7BZ15J0jY+nqZPncoX3lqBagMFeer5bR14ov5g28/6aQrOBd5qSu/LtgX0V2NLIekm4CaApSz3Lm7O0I/Y/cB5TenRbFuCme0EdgKs0og1/0ue9dhYbqdf27s1SY9dsySX56L3RtMNb6TF2omPCw43/TfXUMGvXtSEdKJDW95PU/AUsEnShZKGgeuAXX3sb0HRc401s0lJ3wX+CQwAt5vZizN2ZPOcvtpYM7sfuH+GjmVBESMvJ9x7BTnU1J+eGM99/NYfNifpwU/md/H6NWuS9Pl/H06LeH5P/kv19EyjgXw/ttS4ouQJLmqsEyHWiRDrRLVtrNK2rX7sRC7L2nvTHtu6kdW5PG9emw4Q3rxqVZK+4PWV+bLHJ9L0uvzkTu1UOoiw5Uvz+6mfyaOx/ODl9L7afhL0RYh1IsQ6UW0ba61zpfk+4amj6TYVTKiM/iOdJTv0ubPT76zIz6LZQDo5Pj56di5PfTCdsz6ycTiXZ81zR8/ss9a+XkaNdSLEOhFinQixTlQ/CdPtqrClJy+r509wtTcPJOmRs9KOfP0T+au/tm51kn5vc77zv+z9dIAwdDx/rAOHDp9+r8n2EzJRY50IsU6EWCdmd6Jb+b+ramknXYP5Q9RIvnPftoyMj9ekbaoV/ObLDqUTNUNH8hPxNtg0QV54D0yDqLFOhFgnQqwTIdaJ2R0gWL6Dnbsjs+iq6NvvJMmhicn088H8Fdih1enJa2LlUC7P0XPT2az6+fnZrXUPfdh0cO0HO1FjnQixToRYJ6pvYz2YmOicBobePpykhw8vy+Wpt9g4cmG+qLVPNl2diCsI1RNinQixTlTaxqpWo7b8zCS0hvP9RNaOJEkbzvc3bUnaTz2+Nr0qu/zltJ8LoOPp1d51zxzN5bGWyZuV+/NlN98J04mosU6EWCdCrBNdxUq6XdJBSS80bRuR9ICkPdnPLjPPi48yJ687gNuAPzZt2wHsNrNbs+DkHcAt3XZk9Tr1Y8fObGh+P8UHH6TpgqsBtZXpbZpLtDHNUHQb/Ip0QHD4U92D+T7amK93G19rmvDpZxLGzB4F3m/ZvB24M3t/J/DVbvtZbPTa3VpvZlMX998B1rfLuFhDPvs+eVkjLr/t/4SZ7TSzS83s0iHa3wG90Oi1xr4raYOZHZC0ATg4kwfVlZZOeu1Ey6TLyfzVVbW0hwP5eRoml6Tt+Yn1BYOBgja/iF5r7C7ghuz9DcDfetzPgqVMd+su4F/ARZL2SboRuBXYKmkPcGWWDpro2hSY2fVtPrpiho9lQTH3J7oL+orWElpUO5yfUMl9pyW06OSqfFuplia1Nl6Qp/nCpdN6BUEHQqwTIdaJEOvE3D95lcBarsqqoBNvA+m2yWX5PIMtt8bXJnNZWk6mcSdM5YRYJ0KsEwuijc2t01Jwe33r5Em94AJsK7WCiZp0P+0nZKLGOhFinQixTsz9NrbkxPJ0sYIqpZZuqSYLym6+wzDCkaonxDoRYp0IsU7M7smr1xNTrYfvtV6JKNpFa5aiNdQHytXFqLFOhFgnQqwTs7teQan8M/S3n96zIxpFF30nGSDEJEzlhFgnQqwTc38SpoCii4Vdv1PiKVC5SZjCNtb3bsOgCyHWiRDrRIh1Yu6dvHoZEMzUIKJ1t0XnO+db5YMuhFgnysQgnCfpYUkvSXpR0s3Z9gj77ECZGjsJ/MDMNgOXAd+RtJkzYZ+bgN1Zeu5ilr4K85R4laRMyOcBM/t39v4j4GUaT/OMsM8OTKuNlXQB8FngCaYR9rkYKS1W0krgXuB7Znak+bNOYZ+SbpL0tKSnJzhZlGVBUkqspCEaUv9kZn/NNr+bhXvSKexzscbSlukVCPg98LKZ/aLpo0rCPlVT7kWtlr7mIGVGXl8EvgU8L+k/2bYf0wjzvCcLAX0DuNblCOcpZUI+H6f97V8R9tmGufl/tAAIsU6EWCdCrBMh1om5N9HtRYmrtDNJ1FgnQqwTIdaJEOtEiHUixDoRYp0IsU6EWCdCrBMh1okQ60SIdSLEOhFinQixToRYJxbPFYRe6GOdn6ixToRYJ0KsE4unja3HVdoFQYh1IsQ6sXja2DLM4PqUUWOdCLFOhFgnykTNLJX0pKT/ZrG0P822XyjpCUl7Jf1ZUsHzphcvZWrsSeByM/sMcAmwTdJlwM+BX5rZp4EPgBvdjnIeUiaW1sxs6oFZQ9nLgMuBv2TbI5a2hbKRiQNZjNdB4AHgNeBDM5t6Gss+GoHLRd+NkM92mNkpM7sEGAW2ABeXLSBCPktgZh8CDwNfAFZLmhpgjAL7Z/bQZh9T/lWWMr2CtZJWZ++XAVtprFnwMPD1LFs8QrWFMkPaDcCdkgZo/CHuMbP7JL0E3C3pZ8CzNAKZg4wysbTP0Vj8oXX7GI32NiggRl5OhFgnQqwTIdaJEOvE/LyCUO9hifiKiRrrRIh1IsQ6EWKdCLFOhFgnQqwTIdaJ+TlAqIq4VX7uEWKdCLFOhFgnQqwTIdaJEOtEiHUixDoRYp0IsU6EWCdCrBMh1okQ60SIdSImuqdLydXpo8Y6EWKdmM6j/QYkPSvpviwdIZ8dmE6NvZlGtMwU8yvks8zjU2eQspGJo8CXgN9laREhnx0pW2N/BfwQmLox9Rwi5LMjZQLovgwcNLNneilgsYZ8ln0Y5VckXQ0sBVYBvyYL+cxq7YIM+eyHMmH1PzKzUTO7ALgOeMjMvkmEfHakn37sLcD3Je2l0eZGyGcT0xrSmtkjwCPZ+wj57ECMvJwIsU6EWCdCrBMh1okQ60SIdSLEOhFinQixToRYJ0KsEyHWiRDrRIh1IsQ6EWKdCLFOhFgnQqwTIdaJEOtEiHUixDoRYp0IsU6EWCdCrBMh1okQ60SIdSLEOhFinQixToRYJ0KsE7IK4kpPFyYdAt4A1gD/q6zg/uh0rBvNbG3RB5WKPV2o9LSZXVp5wT3Q67FGU+BEiHVitsTunKVye6GnY52VNnYxEE2BE5WKlbRN0ivZOjI7qiy7DJJul3RQ0gtN20YkPSBpT/bz7DL7qkxs9sDg3wBXAZuB6yVtrqr8ktwBbGvZtgPYbWabgN1ZuitV1tgtwF4zGzOzceBuYHuF5XfFzB4F3m/ZvJ3GmjcwjbVvqhR7LvBWU7rtOjJzjPVmdiB7/w6wvsyX4uQ1DazRhSrVjapS7H7gvKb0fFlH5l1JGwCynwfLfKlKsU8Bm7IV5oZprC+zq8Lye2UXjTVvYDpr35hZZS/gauBV4DXgJ1WWXfL47gIOABM0zgE30ljvZjewB3gQGCmzrxh5OREnLydCrBMh1okQ60SIdSLEOhFinQixTvwf0qepT5DMLBgAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "plt.imshow(np.array(a), vmin=0.0)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "d58be088",
      "metadata": {
        "id": "d58be088"
      },
      "source": [
        "### Labeled attention plots"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dc11a42d",
      "metadata": {
        "id": "dc11a42d"
      },
      "outputs": [],
      "source": [
        "#Labeled attention plots\n",
        "def plot_attention(attention, sentence, predicted_sentence):\n",
        "  sentence = tf_lower_and_split_punct(sentence).numpy().decode().split()\n",
        "  predicted_sentence = predicted_sentence.numpy().decode().split() + ['[END]']\n",
        "  fig = plt.figure(figsize=(10, 10))\n",
        "  ax = fig.add_subplot(1, 1, 1)\n",
        "\n",
        "  attention = attention[:len(predicted_sentence), :len(sentence)]\n",
        "\n",
        "  ax.matshow(attention, cmap='viridis', vmin=0.0)\n",
        "\n",
        "  fontdict = {'fontsize': 14}\n",
        "\n",
        "  ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
        "  ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n",
        "\n",
        "  ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
        "  ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
        "\n",
        "  ax.set_xlabel('Input text')\n",
        "  ax.set_ylabel('Output text')\n",
        "  plt.suptitle('Attention weights')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1e11e85e",
      "metadata": {
        "id": "1e11e85e",
        "outputId": "e6333bbe-17e0-4fbb-b917-2f8cebc97103"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\Nhung\\AppData\\Local\\Temp/ipykernel_31496/2069917776.py:14: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
            "  ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
            "C:\\Users\\Nhung\\AppData\\Local\\Temp/ipykernel_31496/2069917776.py:15: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
            "  ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoEAAAJcCAYAAACG835JAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABBa0lEQVR4nO3dd5wlVZ338c93mGFgQFAQFXUBFxOIijoG1oQ5B5TVXQMCLiOKu4YHZdWHZ9lowHXV1RXBAOsaERMGDEsQEBOuCpIWcEAlSJAwxGHm9/xR1XK59PR0N9Nd90593q9Xv+bWqbp1f7dp+n771DmnUlVIkiSpXxZ0XYAkSZLmnyFQkiSphwyBkiRJPWQIlCRJ6iFDoCRJUg8ZAiVJknrIEChJ05DkkCQHdl3HZJI8PsnZ0zx21yS/neuaJI0+Q6CkkZXk+CR/SLJ4qH15kqcObG+XpJIsXEevu2eSkwbbqmrfqvrHdXH+da2qTqyqB6yLcyU5PMk/rYtzSRpthkBJIynJdsDjgQKe3201krT+MQRKGlV7AD8EDgdeNdGY5FPANsDRSVYkeSvw/Xb3VW3bLu2xeyc5s+1N/HaSbQfOU0n2TfK/Sa5K8uE0dgAOAXZpz3VVe/xtesiS7JPk3CRXJvlaknuu7dzDbzDJRkluSHLXdvsdSW5Jslm7/Y9J3t8+XpzkvUkuTHJpe3l643bfbS7xJnl4kv9Jcm2SI5N8frh3L8n/SfL7JBcn2attWwa8HHhr+96PbtsPSPK79nxnJ3nK9P8zShpVhkBJo2oP4NPt1zOS3B2gql4JXAg8r6o2rar3AE9on3Pntu2UJC8A3g68CNgKOBH47NBrPBd4JPAQ4CXAM6rqTGBf4JT2XHceLizJk4F3ts/ZGrgA+Nzazj18nqq6EfgJ8MS26YntuR47sH1C+/hdwP2BnYH7AvcC/t8ktW0IfJkmPG/Rvufdhg67B7B5e45XAx9OcpeqOpTm+/2e9r0/L8kDgNcDj6yqO7XvY/nw60oaP4ZASSMnyeOAbYEvVNWpwHnAy2Z4mn2Bd1bVmVV1C/AvwM6DvYHAu6rqqqq6EDiOJmBNx8uBT1TVz6rqJuBtND2H283i3CcAT2zHMz4E+GC7vRFNiPx+24u4DHhTVV1ZVde27+cvJjnfY4CFwAeramVVfQn48dAxK4F/aPd/E1gBrGlM4SpgMbBjkkVVtbyqzlvTN0bS+DAEShpFrwK+U1WXt9ufYeCS8DRtC3ygvRx7FXAlEJrerwmXDDy+Hth0mue+J02PHQBVtQK4YpbnPgHYFXg4cBrwXZoewMcA51bVFTQ9mUuAUwfezzFt+2S1/a6qaqDtN0PHXNEG47XWV1XnAm8EDgJ+n+Rzg5e+JY0vQ6CkkdKOc3sJTW/YJUkuAd4EPDTJQ9vDauhpw9vQBJ/XVNWdB742rqofTKOMyc436CKakDlR8ybAlsDvpnHuYT+g6YXbDTihqs6gGfP4bG69FHw5cAPwoIH3snlVTRbcLgbuNTQG8U9mUM/t3ntVfaaqJnpnC3j3DM4naUQZAiWNmhfSXILckeYS6s7ADjRj+vZoj7kU+NOB51wGrB5qOwR4W5IHASTZPMmfT7OGS4F7t+PrJvNZYK8kO7fL1/wL8KOqWj7N8/9RVV0PnArsx62h7wc0l7NPaI9ZDRwG/FuSu7Xv515JbjfOEDiF5vv3+iQL27GRj5pBSbf53iZ5QJInt+/zRpowunoG55M0ogyBkkbNq4BPVtWFVXXJxBfwIeDl7di5dwL/t700un8bpP4ZOLlte0xVfZmmx+pzSa4BTgeeNc0ajgV+BVyS5PLhnVX1PeBA4CianrftmXx83nSdACzi1rF7JwB34tZZzwAHAOcCP2zfz/eYZBxfVd1MMxnm1cBVwCuArwM3TbOWj9OM/7sqyVdoxgO+i6Y38hLgbjRjICWNudx22IgkaX2T5EfAIVX1ya5rkTQ67AmUpPVMkicmuUd7OfhVNLOOj+m6LkmjZZ3cYkmSNFIeAHwB2AQ4H9i9qi7utiRJo8bLwZIkST3k5WBJkqQeMgRKkiT1kCFQkiSphwyBkiRJPWQIlCRJ6iFDoCRJUg8ZAiVJknrIEChJktRDhkBJkqQeMgRKkiT1kCFQkiSphwyBkiRJPWQIlCRJ6iFDoCRJUg8ZAiVJknrIEChJktRDhkBJkqQeMgRKkiT1kCFQkiSphwyBkiRJPWQIlCRJ6iFDoCRJUg8ZAiVJknrIEChJktRDhkBJkqQeMgRKkiT1kCFQkiSphwyBkiRJPWQIlCRJ6iFDoCRJUg8ZAiVJknrIEChJktRDhkBJkqQeMgRKkiT1kCFQkiSphwyBkiRJPWQIlCRJ6iFDoCRJUg8ZAiVJknpoYdcF9FGSF83iad+qqhvWeTGSJKmXUlVd19A7SVbP8CkF3K+qzp+LeiRJUv/YE9ide1TV76dzYJJr57oYSZLUL44J7MYRwEwu7f4XcM0c1SJJknrIy8GSJEk9ZE9gR5KsSnK3ruuQJEn9ZAjsTrouQJIk9ZchUJIkqYecHdytlySZcsJHVf3nfBUjSZL6w4khHWnXCryeZg3ANamq2myeSpIkST1iCOxIGwKnvVagJEnSuuSYwO6YviVJUmcMgd1xdnAHkqxul+eZ7Ou6JL9I8jdd1ylJ0lxzYkh3prxrSJKlwD9V1TPnr6ReeD1wEPBl4Edt26OBFwLvBv4EeFeSqqp/76JAjYYk/28Nuwq4ETgXOKaqZnL3H0k9lWSLmT6nqq6ci1omOCawQ0meBjwdWAl8rKrOT3J/4GDgucB3DYHrVpKvAl+rqo8Ptb8aeH5VvSDJvsBfV9WDOilSIyHJacA2wCbARW3zPYHrgMto/mD4PfDEqjq/kyIljY12LsBMQlcB95/L3y+GwI4keRXwSeBKYAuaD5U3AB8FvgS8r6pO667C9VOSFcDOVXXuUPt9gV9U1SZJtgdOq6olnRSpkZBkT+AVwJ5V9du27d7AJ2ju5/0N4AvAtVX1wo7KlDQm2hD4YprP/bUeDnwT2GkuQ6CXg7vzJuDtVfWuJC8BPge8BXh4VZ3XbWnrtStoLv2+d6j9hcDl7eNNgavnrySNqL8DXjARAAGq6rdJ3gp8par+M8k7gK92VqGkcXIB8P2qumI6Byc5n+ZK4ZwxBHZne+Dz7eMvAquANxsA59zfA4cleTLw47btkTSX5fdpt58GnNBBbRotdwc2mqR9MTBx3+9LAXuMJa1VVd1nhsfvNFe1TDAEdmcTmrFFVNXqJDcCv+m2pPVfVX0iyZnAXwPPb5vPAh5fVT9sjxnuJVQ/fQ/4aJJlwKlt2yOAjwDfbbcfDPy6g9ok6Q4zBHbrOUkmLjsuAJ6R5NLBA6rqS/Nf1vqtqk4BTum6Do28vwL+k2YW+aq2bQHwHW7tNb4W2H/+S5PWjST3ABYODnto2+8NrKyqSyd/pmYrSYA9aMYH/inNBJDzgSOBT9c8TtZwYkhH2gGia1NVtcGcF9NDSe5Jc0nvNmtlVtXPuqlIoyrJA4AHtJtnVdU5XdYjrUtJvgd8vqoOG2p/NfDSqnp6N5Wtv5J8iWYc+mnAGTSTQHYEdgK+XFUvnrdaDIHqkyQPo5nZ+UBuv2C3oVtSryS5Cnh0VZ091H5/4IdVNeO17bRmSV5OswrIi6rqO0P7ngEcBSyrqs/MRz1eDh5hSZ5aVd/ruo71zKE0Yy/3oVn7zb+CtEZJXgo8hcl7jZ8/6ZOk8bKQZrLTsI3W0K475hXAu4cDIEBVfTvJwe0xhsA+SnIvYC9gb2BbwJ6pdWtH4GFe0tPatL+M3wgch38wzJskC6vqlq7r6JEfAa9tvwbtB/xk/stZ7z0U+L9T7P8G8Jp5qsUQOAqSbAC8AHg1zVIlvwQOoRkkqnXrNOAegCFQa7MH8JdV9cWuC+mZi5McAXy8qs7supiZGsPxxu8Ajk3yEODYtu3JwMOAp3ZW1fprS+DiKfZfTHMDiXnhmMAOtQPO/4rmw+Y6mu7fA4CHVtUZXda2vmrXB/wXmr/ETmNoIc65vk+jxkeSy4Bdhu8uo7mVZB+aqyGPplnL82M0ExdWdFrYWozzeOMkDwXeCuzcNv0PcHBV/aKzotZT7aTQu1fVZWvYf3fgovn6eTEEdiTJiTQzgY4CPlVVJ7TtKzEEzpmhWdmDP/xhxH9Ra34l+WeaJTIO6rqWPkqyA82wmFfQ3MXnSJrewZM7LWwNkvyE5o5E/8Akwweq6oIu6tJoaT+DPgFcv4ZDlgB7GQLXc0luAT4MHFpVvxpoNwTOoSRPnGr/RBiXknwYeBnNEg6/5Pa9xn/TRV190w6XeR1wMLAI+F/g/TS/O6ez1Na8SHIdYzreOMli4OU0Y6YL+BXw2aq6qdPC1kNJjmca44ur6klzX40hsDPtpYO/ovmQWU6zKO1naWaujk0IHMPxL9K0JDluit1VVU+et2JmYdw/2JNsCLyIpjfwycBJwMeBewJ/A5xYVX/RXYW3leSHwFur6vtd1zITSXYEjgE2oxkiA82dcK4GnjmO4zI1fYbAjiXZCPhzml90j6MJU38LfKyq/tBlbVMZ8/Evd6eZ+Tb44fgRV8bX+qL9YP8WsDlj9sGe5OE0vw//kqb39T9pfh+eM3DMg4CfVtXG3VT5xzoGB/DvzBiON07yXZpLk6+sqmvats1ofr8vrqpndFmf5pYhsCNJtgF+M3h7mCT35daJIlsCx1bVszoqcUrjOv4lyWNp/uq9lFtvHbcLTW/mM9pbyo2sJHcFtgd+Pi49OuOu/UPtvjQ/4+dV1Y0dl7RW4/zBnmQVza35PgZ8dbLlYpJsAnyoqvaa7/qG6ljN7ccWM0nbyP5hnOR64JGDw5La9gfTLBa9STeVrZ+SvHk6x1XV++a6FjAEdqb9Rbd1Vf1+kn0bAM8F9q6qF8x7cdMwruNfkpxC81f6vhPjiZIsoFmSZ6eq+rMu61uTJHeiuRS2O80HzP2q6vwkhwCXOHlh3UuyiKZn5/XAhjQf5jcB/w68o6pWTvH0To3zB3uSbUf1j8hhaxtjPGhUxxsnuRJ43vCEmySPownhW3ZT2fopya+n2F00S5gtnq8/GlwnsDvDl1D/qKpWAV9tv0bVuK63tzOw5+CA8qpaneR9NMsijKp3A/cCHk4zNmrC14F/Bg7qoKb13btpLknuy63f88cD76QZtrF/R3VNx43AnSdp37zdN7LGJQDC6Aa7GToaOKxdmueHbdsuNLc2+1pnVa2nquo+k7Un+VOa3+V/zjyuEbxg7YdIk3o78J4kT01y9yRbDH51XdwUrgYm+5/wPsBV81vKjDwfeGNV/ZzbXmo6E/jTTipa/70MeHVVHVFV57Vfh9MM2Xh5t6Wt1cQH+2OTbNB+PY4x+GBPsmGSv09yTpIbk6wa/Oq6vjVJ8vokr5ik/RVJXtdFTdP0BpoZ1yfS/IFwI3ACzR/4b+yurH5IsmWS99OsQnA34DHzOeHJnsBu7Z9kygVQq+of5quYGZq4p/F3mGT8C6N7u7vPAR9P8lbgB23bY2l6fT7bWVVrdxeaMZjD7gSM7AfjmNscOG+S9vOYvJdtlLwBOILmg33i52MBTQB8Y0c1Tdc/Ai+l6XH9N+AtwHbAXwAHdlfWWr2R5q5Pw5YDnwT+Yz6Lma6qugp4QZL70Uz0AzjTRdLnVpKNgTfTLNK9HNitqr4173U4JrAb7YDis4Gp7pFZVfWQeSppRsZ1vb122YmDaS7xTfwRtBL4CHBAVd3cVW1TadeW+kpVvT/JtcBDqurXST4CbFtVz+62wrUbt+WE2iU/Tq2q/YbaPwLsXFW7dFPZ9I3jB3s7Zuq1VXVM+7O+c1Wdl+S1wFOqaveOS5xUkhuBB1bV8qH27Wi+953OZNZoaMegvxr4e5rPngNpbhjRSRgzBHakDYH3mGxiiOZekiU0s2yhmfG5ptXbR0KSPwO+TdOT+QqamZMPAh4FPGFUgxSM73JCSZ4AfBP4HbeOlXoMzTp1z6qqk9b0XM1eO6nlgVV1YZKLgedW1alJ7gP8oqo267jESSVZTjNk4ytD7S8CPlBVf9JFXWuT5INT7XdR9HUryRnAtsAHaSaZTTpGd76WFPJycHfWi/Td9u5sQzN78o9GfcHUNvSdttYDR0RV/aANgvvTXI58CvAzmnvbjvr7OJRmEfR9mGQ5oVFVVd9Pcn+aNSUnetOOBP6jqi7qrrLpaWvfncn//9y7k6Km50KaoH0hcC7wDOBUmskKN3RY19p8Bvhgu3LC8W3bk2jubvLpjmqajgcPbS+i+XnfgNGeLDeuJn6XHEBzKXjYvA6psiewI+PeE9iGv88AT6D5gZ34wQVghHt3NqIZL/UUJr80OZKX38fZuC4nNM6SPIfmvuT/AzwC+AlNz/dimjttPL/D8qaU5J3Aiqr65yS704zV/S3N7PiDq+odnRa4Bu2SQv9JM55xcBzmkTTrNY7skkLD2t+TH6f5WTmk63rWJ9NdVmi+hlQZAjuS5O9ofqGN9GXINUnyBZoFrfej+YB5JnB3msWj31RV3+2wvDVK8glgN5pfzJMtcv33XdQ1XeM2rg7G+nZaD59q/4h/z08FvlhV72zH1T2U5uf9U8Ap87UQ7bqQ5NE0k7fOqaqvd13P2rTjMHduN39eVf/bYTmz1t6V5ZhRvYytdcMQ2JEkm9IsCHnFQNsONDPhNgW+VFWf66q+tUlyKfCcqvppkmuApVV1TtsDcWBVPabjEifVLoz6kqr63loPHiHjOq4OIMmTGc/baU3cDWLw+z3yvd0A7aoDD2kXFL+SZtzo6e1i0d+oqm06LnFKaW7t+Fhu/wdPVdVHuqmqX9oeq69U1V26rmV9kmQZcMTEHZ/asH32xJ1x2rvhHFBV/28+6nFMYHc+QrNm3evhj7cDOxFYDVwMfDrJgqr6THclTmlj4PL28ZU0v6zPoVnraJQvqV5PMz5t3IzluLrWuC4nNLye5CLgYcA7gLfNfzkzci2wUfv4Yprb3p1O8zt/pD/U27X2Pkbz8/EHbvszUzS/O0dCO6nibVV13bhOsJjkNmYBtqZZC/Ob81/Reu8jwFeAiaFgp9D0HJ/fbm9K8zvGELie2wV4zcD2K4GbgR2q6uok76YJiKMaAs+i6ZVaDvwc2DfJb2guD/+uu7LW6j3Am5Ps29WU/FnakfEdV/ekrguYjTXcueLcJFcDfwfM+5peM/Aj4HE0f5R9A/jXJA+lGQox0vfHprlrwnuAf6hJ7hs8Yh5M88fBxONx9NdD26uBy2jWNnzn/Jez3hu+krPGu4fNB0Ngd7bmtgvRPgk4qqqubrePAEZ5Bt8HaG4bB804wGNo7rBwE7BHV0VNJsnwHRKeADyznao/fGlyVAfMj+tt+kZ2zcg74NfcOuZrVL2ZpkcBmlsK3gl4Mc3Pz7RuYN+hzYDDxyAAUlVPmuzxOEhyBPCWNd3GTP1gCOzO9cDgTdwfBXx+YPtGYMm8VjQDVTW45ME5NJeb7gtcWFWXT/6szgzfaePLnVQxQ0O335u4Td9YjKtrJ1X8vL0v81hOsJjk9ocTl8kOolnofZS9D/hUkrPayWev7bqgGfg08ByaNdRGWjvRbDqqqia7m0gnknyS5g4sS9qJQ2s04ssJ6Q4yBHbnF8BeNLeO2xXYCjh2YP/2NGO/RlaSN9L0KtyrbboIeF+S94/Spdaq2mvicZJlVXXoZMclGbWlEC7n9mPoxmVc3U9pei5/3z4enmAxYRRrnzD8/YfmPfyGZhmQUXY9zdWElUmOorkjwbj0yL4Z+EqSpzD5HzyjdCvNrYa2n0BzOXVi7c6daCa2jNrM+K1ofpa3oBk7Oi51ry+e0w4rgeb7/Ix2siXM8y0pDYHd+UfgW0leQvM/5OFVdfHA/t2Akb0jQZL3AMtobsE2McZoF5rBrFsz+SKYo+DdSa6oqqMGG9sA+KyOalqTwctL29GEj+H7BC+gWQx41NyHZlzRxOMNmLz21fNZ1AztxW2/5xNjpc7n1qEQI6mqXtbOMtyNZpjGd9u7b3wW+K+qOr3TAqf2Gpolpy6nubowPDFkZEJgVT1v4nGSt9EsZr1XVV3Xtm1Cs97eSC3oXlXPbVd12IfmD5qxqHs98vGh7Q8Pbc9bJ4pLxHSoXRLm6cAlwJFVtXpg3zLgx1X1847Km1K77MSyqvriUPvuwEerastuKpta27vwJeBFVfXfbduhNB86u1bV+VM9vytJVgFbDy8unmRL4PcjvlzJWNY+rnVPJslWNB/2+9Lckm1kOwCS/B54Z1X9W9e1zEQbsp9SVWcMtT8I+O+qGqk/HJKcRXM3lh8yRnVr3Vqw9kO0riV5VJINqurMqvpAVX1+MAACVNWhEwEwySPa1ehHzS/X0DayP1dt8Hs18MUkj05yGM0vwpENgK3b3JFlwKas4d6TI2Rcax/Xum+jvfvDk2l+zu/P6C+RtAEwPJlrHGxKc7u7YVszguO7q+qB7Qz4sap7nE189s/g+Dn/7B/ZvwbXc6fQXE66bG0Hto7jtusIjYL/pFkO5g1D7a+luSvByKqqLya5C814l4uBJ1bV8m6rmtzA2mMFvDPJ4B1mNqCZUPTz+a5rOsa19nGte1CSAE+jWevthTSXtI+k6fE5scPSpuOTNHWPzGXfaToK+GSSt9D0rgE8Bng3zdWHUTWudY+jkfvsNwR2I9z+w2UqG679kLk3tBjqQuAVSSYuJwA8muYvypG6WfoUi7j+nmbMy5ubz8yRXNB1Yu2xADvQrCU54WbgZ8B757uoaRrX2se17kEX0yy18i1gT5q7hNw85TNGxxLgr9rfLb/k9hNDRu3/0QmvBf4VOJxb1w68hWb81/4d1TQd41r3lJKcCdxvxIY+jNxnv2MCO5DkeGY+8PNlQxNH5l2S46Z5aFXVk+e0mBkY17oHtUs6vKGqrum6lpka19rHtW6AJPvQjDO+qutaZmot/7+O7P+jE9pJFdu3m+dNTLYYdeNa95okeT2w5SjdD34UP/sNgZIkST00sgP4JUmSNHcMgZIkST1kCBwh7dqAY2lcax/XumF8ax/XumF8ax/XumF8ax/XumF8ax/XuqG72g2Bo2Vsf4AZ39rHtW4Y39rHtW4Y39rHtW4Y39rHtW4Y39rHtW7oqHZDoCRJUg85O3iGNszi2ohN5uTcK7mJRSyek3OvvvPc1Dxh5U0rWLR40zk59wO3me66mjN32RWr2GrLubn71znnz+2d825eeR0bLpqb/665/qY5OS/AzXUjG2ajOTn30I131rmVdROLMjf/j87l3ULn8nfLXBvX2se1bhjf2se1bpjb2q/lD5dX1VaT7RulRRTHwkZswqMXPLXrMmbs+qc8qusSZu3ED3206xJm5Wkv3avrEmZt4alnd13CrNTN47Ie8u3VqlVdlzA749yRsGBsbv98W3P8x47WL99bfeQFa9rn5WBJkqQeMgRKkiT1kCFQkiSphwyBkiRJPWQIlCRJ6iFDoCRJUg8ZAiVJknrIEChJktRDhkBJkqQeMgRKkiT1kCFQkiSphwyBkiRJPWQIlCRJ6iFDoCRJUg8ZAiVJknrIEChJktRDhkBJkqQeMgRKkiT1kCFQkiSph8YuBCbZI8kVSRYPtX86ydfax69Jcm6Sm9t/9xk6tpLsPtS2PMn+c/8OJEmSujd2IRA4kqbuF0w0JNkc2A34eJLdgA8B7wd2Aj4A/EeS581/qZIkSaNpYdcFzFRV3ZDk08DewBfa5pcB1wDfAE4APlVVH2r3nZPkEcABwNGzec0ky4BlABux5A5UL0mSNBrGsScQ4DDgaUnu3W7vDRxRVbcAOwAnDx1/ErDjbF+sqg6tqqVVtXQRi9f+BEmSpBE3liGwqn4B/AzYM8lOwFLgE2t72tDjDO1ftO4qlCRJGm1jGQJbhwF7An8FnFxVZ7ftZwKPHTr2ccAZA9uXAVtPbCS5++C2JEnS+m7sxgQO+CzwPuC1wL4D7QcDRyY5FfgO8Ezg5cCLBo45FtgvyQ+AVcC/ADfOR9GSJEmjYGx7AqvqWpqJITdx6wQRquorwF8Db6Lp/XsD8LqqGpwU8n+A84HjgS8CHwN+Px91S5IkjYJx7gmE5hLu56vqusHGqjoEOGRNT6qqi4BnDTUfte7LkyRJGk1jGQKT3AV4PPB04KEdlyNJkjR2xjIEAv8DbAG8vapO77oYSZKkcTOWIbCqtuu6BkmSpHE2thNDJEmSNHuGQEmSpB4yBEqSJPWQIVCSJKmHDIGSJEk9ZAiUJEnqIUOgJElSDxkCJUmSesgQKEmS1EOGQEmSpB4yBEqSJPWQIVCSJKmHDIGSJEk9tLDrAsZSVdcVzNidTjy/6xJm7UXnPq3rEmbl/Bcv7rqEWXvAFffuuoTZueB3XVcwa3XDjV2XMEuruy5g1rJoTD8CV63qugKNkyn+F7UnUJIkqYcMgZIkST1kCJQkSeohQ6AkSVIPGQIlSZJ6yBAoSZLUQ4ZASZKkHjIESpIk9ZAhUJIkqYcMgZIkST1kCJQkSeohQ6AkSVIPGQIlSZJ6yBAoSZLUQ4ZASZKkHjIESpIk9ZAhUJIkqYcMgZIkST00diEwydeTHN51HZIkSeNs7ELgupbkoCSnd12HJEnSfOp9CJQkSeqjkQ6BSZYkOTzJiiSXJnn70P67JDkiyR+S3JDke0keNLB/z/a5T0lyepLrkhyX5D4T+4G/Ax6UpNqvPefzPUqSJHVhpEMg8F7gacCLgacADwOeMLD/cODRwAuARwHXA8ck2XjgmMXA24C9gV2AOwOHtPs+D/wrcDawdfv1+Tl5J5IkSSNkYdcFrEmSTYFXA3tX1bfbtr2A37aP7wc8H3hiVX2/bXslcCHwcuBj7akWAvtV1dntMe8FPpEkVXVDkhXALVV1yRS1LAOWAWzEknX+XiVJkubbKPcEbg9sCJwy0VBVK4DT2s0dgNVD+69u9+84cJ6bJgJg66L2vHeZbiFVdWhVLa2qpYtYPNP3IUmSNHJGOQTeETXw+JY17Ftf37skSdJajXIQOg9YCTxmoiHJJsBO7eaZNPXvMrB/M+DBwBkzeJ2bgQ3uaLGSJEnjZGRDYHvp9+PAu5M8rZ31+wnawFZV/wt8FfhokscneTDwX8A1wGdm8FLLgW2TPDzJXZN4vVeSJK33RjYEtvYHjgO+3P57OvD9gf17AT8Gvtb+uwR4ZlXdMIPXOAr4JvDfwGXAX97xsiVJkkbbyM4OBqiq64A92q/J9v8BeNUUzz+cZhmZwbbjgQxs3wTsfoeLlSRJGiOj3hMoSZKkOWAIlCRJ6iFDoCRJUg8ZAiVJknrIEChJktRDhkBJkqQeMgRKkiT1kCFQkiSphwyBkiRJPWQIlCRJ6iFDoCRJUg8ZAiVJknrIEChJktRDhkBJkqQeMgRKkiT10MKuCxhLSdcVzNzKm7uuYNZ+88kduy5hVhb+adcVzN7yF9+16xJmZZtvbdh1CbOW0/636xJmZ3V1XcGsZYMNui5hVsb3Oz7GVq3quoI5YU+gJElSDxkCJUmSesgQKEmS1EOGQEmSpB4yBEqSJPWQIVCSJKmHDIGSJEk9ZAiUJEnqIUOgJElSDxkCJUmSesgQKEmS1EOGQEmSpB4yBEqSJPWQIVCSJKmHDIGSJEk9ZAiUJEnqIUOgJElSDxkCJUmSemgkQmCS45N8qOs6JEmS+mIkQqAkSZLmlyFQkiSph0YyBCZ5SpKrkuyb5PAkX0/yhiS/S/KHJJ9MsmTg+MVJ3p/k0iQ3JvlhkscN7P9hkr8d2P6vJJXkHu32kiQ3DT5HkiRpfTZyITDJ7sCXgWVVdUjb/HhgJ+CpwEuB3YA3DDztPW373sDDgNOAY5Js3e4/Hth14PgnApcPtP0ZcAvw43X6ZiRJkkbUSIXAJMuAjwO7V9UXBnZdA+xbVWdW1XeAI4GntM/ZBHgtcEBVfaOqzgT2BS4F9muffzzwuCQLk9wX2Bz4KPCkdv+uwClVdfOa6kry0yQ/XclN6+4NS5IkdWSUQuALgQ8Dz2yD3qAzqmrVwPZFwN3ax9sDi4CTJ3a2x54C7Ng2nQQsBh5JE/hOAr7HrT2Bu9IExUlV1aFVtbSqli5i8czelSRJ0ggapRD4C+Bi4NVJMrRv5dB2Mb3aC6CqVgCn0vT87QocB/wQ2KbtGXwkU4RASZKk9c0ohcBf0wS0pwOHThIE1+Q84GbgsRMNSTYAdgHOGDjueJoQ+ETg+Kq6EfgR8A4cDyhJknpmlEIgVXU+TVB7JvDR6QTBqroO+Ajw7iTPTrJDu3134D8GDj2eJmRuBvxsoO0VTDEeUJIkaX00UiEQoKrOowlrz6KZvDGdHsEDgM8DnwR+DjyEZmzhxQPHnNT+e+LA+MLjgYV4KViSJPXMwq4LAKiqXYe2zwP+ZIrjDwIOGti+CXhj+7Wm56ygmUAy2HY80wuZkiRJ65WR6wmUJEnS3DMESpIk9ZAhUJIkqYcMgZIkST1kCJQkSeohQ6AkSVIPGQIlSZJ6yBAoSZLUQ4ZASZKkHjIESpIk9ZAhUJIkqYcMgZIkST1kCJQkSeohQ6AkSVIPGQIlSZJ6yBAoSZLUQwu7LmDsBLLBBl1XMWOrr7uh6xJmbaujftV1CbNyty3u3HUJs3bhS+7ddQmzcuGzNuu6hFnbbvmmXZcwOzev7LqC2bvbll1XMCsLVq3uuoRZqyUbdV3C7Kwe3+85Z6x5lz2BkiRJPWQIlCRJ6iFDoCRJUg8ZAiVJknrIEChJktRDhkBJkqQeMgRKkiT1kCFQkiSphwyBkiRJPWQIlCRJ6iFDoCRJUg8ZAiVJknrIEChJktRDhkBJkqQeMgRKkiT1kCFQkiSphwyBkiRJPWQIlCRJ6qE5DYFJDk/y9bl8jXUlyelJDuq6DkmSpPlgT6AkSVIPGQIlSZJ6aN5CYJLFSd6f5NIkNyb5YZLHDR3znCRnt/u/n+QvklSS7QaO2TvJhUmuT3J0ktclqaHzPC/Jqe15fp3kn5NsOLD/bkm+muSGJBck2XvOvwGSJEkjZD57At8DvBTYG3gYcBpwTJKtAZJsA3wJ+AbwUOCD7XP+KMkuwMeADwM7A18D/n7omGcAnwY+BDyofb3dgX8ZOOxw4L7AU4EXAnsA262TdylJkjQG5iUEJtkEeC1wQFV9o6rOBPYFLgX2aw97LXB+Vb25qs6uqi8Chwyd6m+A71TVu6vqnKo6DPjy0DHvAA6uqk9W1XlVdRxwALBvGvcHngUsq6qTq+p/gFcBG09R/7IkP03y05V10x36XkiSJI2C+eoJ3B5YBJw80VBVq4BTgB3bpgcCPxl63o+Gth8I/HgtxzwCeEeSFRNfwGeATYB7ADsAqwfPU1UXABetqfiqOrSqllbV0kVZvMY3KUmSNC4Wdl0AUGs/ZEYW0FwiPnKSfZfN4etKkiSNjfnqCTwPuBl47ERDkg2AXYAz2qazgKVDz3vU0PZZwCPXcszPgAdW1bmTfN3SnmPB4PPa8Yj3nPnbkiRJGk/z0hNYVdcl+Qjw7iSXA78G3gTcHfiP9rBDgDcneS9wGM2kjtdMnKL994PASUneAnwFeAKw29DL/QPw9SQXAF8AbgF2Ah5VVW+tqrOTHAN8NMky4Abgfe2/kiRJvTCfs4MPAD4PfBL4OfAQ4JlVdTH8cVzei4HnA7+gCYkTM39vbI85BdiHZoLIL2lm9r57Yn97zLeB5wBPohn392Pgb4ELB2rZkyaIHgscTTNmcPk6fK+SJEkjbU57Aqtqz4HHNwFvbL/WdPzXgT/eZi7JG4BrgN8PHPMJ4BMDx/wbcO7Qeb4DfGeK17mUJmwO+tgUb0WSJGm9MgoTQ/4oyX40M4QvAx4DHAgcXlU1cMxbgO8CK2jW+dsXePv8VytJkjS+RioE0izg/HZgS+C3NOME/2HomKXA/sDmNJd03wZ8YB5rlCRJGnsjFQKr6k00YwGnOual81SOJEnSems+J4ZIkiRpRBgCJUmSesgQKEmS1EOGQEmSpB4yBEqSJPWQIVCSJKmHDIGSJEk9ZAiUJEnqIUOgJElSDxkCJUmSesgQKEmS1EOGQEmSpB5a2HUBY6egVq3quopZGMeaG6tWjGftueHGrkuYtXsfs6TrEmblskfepesSZi2bjOf3vDa4qesSZu3me4/nz8vqhem6hFm7ZtsNuy5hVu76yxVdlzAn7AmUJEnqIUOgJElSDxkCJUmSesgQKEmS1EOGQEmSpB4yBEqSJPWQIVCSJKmHDIGSJEk9ZAiUJEnqIUOgJElSDxkCJUmSesgQKEmS1EOGQEmSpB4yBEqSJPWQIVCSJKmHDIGSJEk9ZAiUJEnqIUOgJElSD3UaApNUkt27rEGSJKmP1hoCkyyeTtu4SnJ4kq93XYckSdJ8mk5P4CnTbJMkSdKYWGMITHKPJI8ANk7ysCQPb792BZZM5+RJnpnkxCR/SHJlkm8n2WENx27XXh7+iyQnJLkhyf8keUiSnZL8IMl1SU5Kcp+h574myblJbm7/3WeS/eckuTHJ5W0dC5McBLwKeE772tW+P0mSpPXawin2PQPYE7g38K9A2vZrgLdP8/ybAO8HfglsDPxf4OgkO1bVzWt4zt8DbwLOBz4CfBb4PfCO9t8jgA8CzwNIshvwofY532nr/o8kl1TV0UmWAh+mCXsnAXcGnty+1nuBHYAtgFe2bVdO871JkiSNrTWGwKo6AjgiyYur6qjZnHz4eUn2ogmRj6IJZJN5X1V9sz3+X4GjgQOr6ri27UM0oW/C/sCnqmqi7Zy2B/OA9rnbANcBX6uqa4ELgF+0x65IcgNwU1Vdsqb3kWQZsAxgo+l1gkqSJI206YwJfGGSzSc2kmyb5L+nc/Ik2yf5TJLzklwDXNq+5jZTPO2XA48vbf89bahtkyQTaWwH4OShc5wE7Ng+/i5N8Pt1kk8neVWSO02n/glVdWhVLa2qpYtYb+bESJKkHptOCDwJ+FGSZ7dj7b5Lc4l3Or4ObAW8Bng08DDgFmDDKZ6zcuBxTdG2ttoLoO39ezjwEuBC4G3AWUnuOY36JUmS1ktTjQkEoKo+muRXwHHA5cDDprp0OiHJlsADgdcNXMp9+HRec4bOBB4LfHyg7XHAGRMbVXULcCxwbJK/oxlb+FzgUOBmYIN1XJMkSdJIW2sgS/JK4EBgD+AhwDeT7FVVv5j6mfyBJjTuk+Q3wL2Ag2l6Atelg4Ejk5xKMzHkmcDLgRe19T8X2B74Ps2kjycBd6IJjwDLgWcleQBwBXB1VQ32PEqSJK13pnM5+MXA46rqs1X1NmBfmhm6U6qq1cBLaYLj6TQzdA8Ebpp9uZO+zleAv6aZHXwG8Aaa3sej20OuAl4IfA84i2YiyV9V1Ynt/sNoAuFPgctoehUlSZLWa9O5HPxCgCRLqur6qvpxkkdN5+RVdSyw01DzpgP7M/B4ObcuQzPR9tNJ2o6ZpO0Q4JA11HASTe/fmmq8DHj6FG9DkiRpvTOd28btkuQMml40kjyU6U8MkSRJ0giazuXg99MswHwFQDsW8AlzWJMkSZLm2HRCIFX1m6GmVXNQiyRJkubJdJZr+U2SPwMqySKaiRdnruU5kiRJGmHT6QncF9iPZomX3wE7A6+bw5okSZI0x6bTE/iAqnr5YEOSx3L7W7VJkiRpTEynJ/Dfp9kmSZKkMbHGnsAkuwB/BmyV5M0DuzbD26xJkiSNtakuB29Is7DzQprbrE24Bth9LouSJEnS3FpjCKyqE4ATkhxeVRfMY02SJEmaY2sdE2gAlCRJWv9Ma7FoSZIkrV+mc+/gx06nTZIkSePDJWIkSZJ6yCViJEmSesglYiRJknrIJWJmo6rrCvqlVnVdwazU6vGsG2DBhRd3XcKsbHGnjbouYdZWb75J1yXMSt3tzl2XMGtX7DiePy8bX7m66xJmbdH14/n5ucFlV3ddwpyYzr2DD09yu/9qVfXkOahHkiRJ82A6IXD/gccbAS8GbpmbciRJkjQf1hoCq+rUoaaTk/x4juqRJEnSPFhrCEyyxcDmAuARwOZzVpEkSZLm3HQuB58KFBCay8C/Bl49l0VJkiRpbk3ncvB95qMQSZIkzZ/pXA7eCHgd8DiaHsETgUOq6sY5rk2SJElzZDqXg/8TuJZbbxX3MuBTwJ/PVVGSJEmaW9MJgTtV1Y4D28clOWOuCpIkSdLcWzCNY36W5DETG0keDfx07kqSJEnSXJtOT+AjgB8kubDd3gY4O8lpQFXVQ+asOkmSJM2J6YTAZ855FZIkSZpX0wmB/1RVrxxsSPKp4TZJkiSNj+mMCXzQ4EaShTSXiCVJkjSm1hgCk7wtybXAQ5Jck+TadvtS4KvzVqEkSZLWuTWGwKp6Z1XdCTi4qjarqju1X1tW1dvmsUZJkiStY9MZE/itJE8Ybqyq789BPZIkSZoH0wmBbxl4vBHwKOBU4MlzUpEkSZLm3FpDYFU9b3A7yZ8A75+rgiRJkjT3pjM7eNhvgR3WdSGSJEmaP2vtCUzy70C1mwuAnYGfzWFNkiRJmmPT6Qn8Kc0YwFOBU4ADquoVc1rVFJLskeSKJIuH2j+d5Gvt49ckOTfJze2/+wwdW0l2H2pbnmT/uX8HkiRJ3ZtOCPw8t4bAo6rq5Lktaa2OpKn7BRMNSTYHdgM+nmQ34EM04xZ3Aj4A/EeS593+VJIkSf20xsvB7Z1B/gXYG7gACPAnST4JvKOqVs5PibdVVTck+XRb1xfa5pcB1wDfAE4APlVVH2r3nZPkEcABwNGzec0ky4BlABux5A5UL0mSNBqm6gk8GNgCuE9VPaKqHg5sD9wZeO881DaVw4CnJbl3u703cERV3UIzaWW4t/IkYMfZvlhVHVpVS6tq6SIWr/0JkiRJI26qEPhcYJ+qunaioaquAV4LPHuuC5tKVf2CZnLKnkl2ApYCn1jb04YeZ2j/onVXoSRJ0mibKgRWVdUkjau4baDqymHAnsBfASdX1dlt+5nAY4eOfRxwxsD2ZcDWExtJ7j64LUmStL6bKgSekWSP4cYkrwDOmruSpu2zwD1oeiY/PtB+MPDKJPsluV+SvwZeDrxn4Jhjgf2SLE3yMOBw4Mb5KVuSJKl7U60TuB/wpSR708wMhuay68Y0M3E7VVXXJvkCsDu3ThChqr7SBr/9aWYIXwC8rqoGJ4X8H5rgeDxwKfBWXABbkiT1yBpDYFX9Dnh0kicDD2qbv1lV/z0vlU3P1sDnq+q6wcaqOgQ4ZE1PqqqLgGcNNR+17suTJEkaTdO5d/CxNJdPR0aSuwCPB54OPLTjciRJksbOWkPgiPofmuVr3l5Vp3ddjCRJ0rgZyxBYVdt1XYMkSdI4m85t4yRJkrSeMQRKkiT1kCFQkiSphwyBkiRJPWQIlCRJ6iFDoCRJUg8ZAiVJknrIEChJktRDhkBJkqQeMgRKkiT1kCFQkiSphwyBkiRJPWQIlCRJ6qGFXRcwlpKuK5i5jG/ez4Ix/H4DWTi+/3tli7t0XUL/jOPvFeDGu27UdQmzVmP6v+jGl63suoRZW3TNzV2XMCu1cIOuS5gT45sMJEmSNGuGQEmSpB4yBEqSJPWQIVCSJKmHDIGSJEk9ZAiUJEnqIUOgJElSDxkCJUmSesgQKEmS1EOGQEmSpB4yBEqSJPWQIVCSJKmHDIGSJEk9ZAiUJEnqIUOgJElSDxkCJUmSesgQKEmS1EOGQEmSpB4a2xCY5KAkp3ddhyRJ0jga2xAoSZKk2essBKbx1iTnJbkhyWlJXjGw/11Jzm73LU/yniQbtfv2BP4OeFCSar/2bPdtnuTQJL9Pcm2SE5IsHTjvnklWJHlKktOTXJfkuCT3md/vgCRJUncWdvja/wTsDuwHnA3sAhyW5A9V9Q3gOmBv4HfAjsAhwE3AgcDngZ2A5wK7tue7OkmAbwBXt/uuBF4FHJvkAVV1cXvsYuBt7flvBI5oz/+MOXy/kiRJI6OTEJhkE+DNwNOr6sS2+ddJHkUTCr9RVf848JTlSf4F2B84sKpuSLICuKWqLhk475OBnYGtquqGtvnAJM8DXgm8p21bCOxXVWe3z3sv8IkkqaqapN5lwDKAjViyDr4DkiRJ3eqqJ3BHYCPgmCSDoWsRsBwgye7AG4H7ApsCG7RfU3kEsAS4rOkU/KONgO0Htm+aCICti4ANgbvQ9B7eRlUdChwKsFm2uF1IlCRJGjddhcCJsYjPAy4c2rcyyWOAzwF/D7wJuAp4PvDeaZz3UuDxk+y7ZuDxLUP7JoKdE2UkSVIvdBUCz6AZ37dtVR07vLPtBfzd4CXhJNsOHXYzt+8Z/Blwd2B1VZ2/bkuWJElaf3QSAqvq2nYc3nvbyRzfp7nk+xhgNXAOcK8kLwdOoZmw8ZdDp1kObJvk4TS9idcC3wNOBr6a5K3AWcA9gGcC3xsYfyhJktRrXV7+PBA4iGayx6+A7wIvBn5dVUcDBwPvB34JPA34f0PPPwr4JvDfwGXAX7aTOp4NHAscRjPr+AvAA2jG/UmSJAnIJJNhNYXNskU9esFTuy5j5jK+wx2zIGs/aARlYZcrMN0xC+55j65LmJWVW9+56xJmbYMVN3VdwqzccM9Nuy5h1q6636KuS5iVLX81nj8rAIuuubnrEmZlwR9WdF3CrH37fw8+taqWTrZvfJOBJEmSZs0QKEmS1EOGQEmSpB4yBEqSJPWQIVCSJKmHDIGSJEk9ZAiUJEnqIUOgJElSDxkCJUmSesgQKEmS1EOGQEmSpB4yBEqSJPWQIVCSJKmHDIGSJEk9ZAiUJEnqIUOgJElSDy3suoCxVNV1BTNXq7quYNZqddcVzE6tGt/vORdd0nUFs7Jo5S1dlzB7CzfouoJZWXTnjbouYdZWbrqo6xJmZcW9Nuy6hFlbvc141n63Y6/quoQ5YU+gJElSDxkCJUmSesgQKEmS1EOGQEmSpB4yBEqSJPWQIVCSJKmHDIGSJEk9ZAiUJEnqIUOgJElSDxkCJUmSesgQKEmS1EOGQEmSpB4yBEqSJPWQIVCSJKmHDIGSJEk9ZAiUJEnqIUOgJElSDxkCJUmSeqjTEJhkQZKPJrkiSSXZdRrPqSS7r2lbkiRJa7ew49d/NrAXsCtwPnBlp9VIkiT1RNch8L7AxVX1g47rkCRJ6pXOLgcnORz4N2Cb9pLu8iTPTHJikj8kuTLJt5PsMMPz3ivJ59pz/CHJN5Lcb2D/QUlOT/IXSc5Lcm2SryS56zp+i5IkSSOryzGBbwD+AfgtsDXwSGAT4P3Ao2guEV8NHJ1kw+mcMMkS4DjgRuCJwC7AxcD32n0TtgNeCuwGPB14GPDPd/D9SJIkjY3OLgdX1dVJrgVWVdUlbfNRg8ck2Qu4hiYUnjSN0/4FEGCvqqr2HK8Bfg88F/hCe9xCYM+quro95lCasYmTSrIMWAawEUvWdJgkSdLY6HpM4G0k2R74R+DRwFY0PZULgG2meYpHAPcBrk0y2L4E2H5g+4KJANi6CLjbmk5aVYcChwJsli1qmrVIkiSNrJEKgcDXaS4Pvwb4HXALcAYwrcvBNIHx5zQ9gsMGZx6vHNpXuGaiJEnqkZEJgUm2BB4IvK6qjmvbHs7MavwZ8JfA5VV11TovUpIkaT0xSr1ffwAuB/ZJct8kTwQOoekNnK5PA5cCX03yxCT3SfKEJP86OENYkiSp70YmBFbVapoZuw8BTgc+DBwI3DSDc1wPPIFm4ekjgbOAI4C70IRMSZIkAWkn0WqaNssW9eg8pesyNA5uOzlprCxYvLjrEmZlwVZjvNznwg26rmBWbt5mi65LmLWLHrtx1yXMyqa/Hd/P7dXj+WPO3Y79bdclzNoxy//t1KpaOtm+kekJlCRJ0vwxBEqSJPWQIVCSJKmHDIGSJEk9ZAiUJEnqIUOgJElSDxkCJUmSesgQKEmS1EOGQEmSpB4yBEqSJPWQIVCSJKmHDIGSJEk9ZAiUJEnqIUOgJElSDxkCJUmSesgQKEmS1EMLuy5AktaZlSu7rmD2xrT2RRdd3XUJs7bh1Rt3XcKsrB7jT+5r7tN1BbOz1Y+XdF3CnLAnUJIkqYcMgZIkST1kCJQkSeohQ6AkSVIPGQIlSZJ6yBAoSZLUQ4ZASZKkHjIESpIk9ZAhUJIkqYcMgZIkST1kCJQkSeohQ6AkSVIPGQIlSZJ6yBAoSZLUQ4ZASZKkHjIESpIk9ZAhUJIkqYcMgZIkST1kCJQkSeohQ6AkSVIPGQIlSZJ6yBAoSZLUQwu7LmAcJFkGLAPYiCUdVyNJknTH2RM4DVV1aFUtraqli1jcdTmSJEl3mCFQkiSphwyBkiRJPWQIbCV5fZKzuq5DkiRpPhgCb3VX4AFdFyFJkjQfDIGtqjqoqtJ1HZIkSfPBEChJktRDhkBJkqQeMgRKkiT1kCFQkiSphwyBkiRJPWQIlCRJ6iFDoCRJUg8ZAiVJknrIEChJktRDhkBJkqQeMgRKkiT1kCFQkiSphwyBkiRJPWQIlCRJ6iFDoCRJUg8ZAiVJknpoYdcFjJssWMCCJZt0XcaMZcMNuy5h9rbaousKZqU2XNR1CbNWizfouoRZuX6rJV2XMGtLzryk6xJmJdff2HUJs3a3U1d0XcKsVNJ1CbO26e/G9Pfi6tVdVzAn7AmUJEnqIUOgJElSDxkCJUmSesgQKEmS1EOGQEmSpB4yBEqSJPWQIVCSJKmHDIGSJEk9ZAiUJEnqIUOgJElSDxkCJUmSesgQKEmS1EOGQEmSpB4yBEqSJPWQIVCSJKmHDIGSJEk9ZAiUJEnqIUOgJElSD3USApMcn6Tar8d0UcNALcsHarlrl7VIkiTNly57Aj8JbA2cCjAQxIa/9m3379pun5Vk4eCJ2iC3/8D2YMi8OcnFSY5J8ookGarjkcCL5/atSpIkjZYuQ+D1VXVJVa0caNuHJhgOfh0x9LxtgVdP4/wTIfNPgecDpwAfBb6cZIOJg6rqMuDK2b4JSZKkcbRw7YfMq6uq6pK1HPNB4KAk/1VV101x3PUD5/ot8JMkPwSOAfagCYmSJEm9NI4TQ/4dWAm8eaZPrKpvA6fh5V9JktRzoxYCP5VkxdDXg4eOuRE4EHhLkq1m8Rpn0FwinrYky5L8NMlPb64bZ/GSkiRJo2XUQuBbgJ2Hvs6e5LhPActpwuBMBaiZPKGqDq2qpVW1dMNsNIuXlCRJGi2jNibwkqo6d20HVdXqJH8LfCXJB2b4GjsC58+qOkmSpPXEqPUETltVfRM4Gfjn6T4nyTOAnYAvzlVdkiRJ42DUegLvnOQeQ20rqmrFGo5/K/BDmokiw5a051pIs1TMs9vjvwr81zqqV5IkaSyNWk/gYcDFQ19/u6aDq+onNL16iyfZvVf7/POBo4FdgH2B3apq1botW5IkabyMTE9gVQ3fyWN4//E0kzqG218KvHSobdd1WZskSdL6psuewGXtEjCP7LAGkvwK+FaXNUiSJM23rnoCXw5s3D7+TUc1THg2sKh97O3jJElSL3QSAqvqd1287mSq6oKua5AkSZpvozYxRJIkSfPAEChJktRDhkBJkqQeMgRKkiT1kCFQkiSphwyBkiRJPWQIlCRJ6iFDoCRJUg8ZAiVJknrIEChJktRDhkBJkqQeMgRKkiT1kCFQkiSph1JVXdcwVpJcBlwwR6e/K3D5HJ17ro1r7eNaN4xv7eNaN4xv7eNaN4xv7eNaN4xv7eNaN8xt7dtW1VaT7TAEjpAkP62qpV3XMRvjWvu41g3jW/u41g3jW/u41g3jW/u41g3jW/u41g3d1e7lYEmSpB4yBEqSJPWQIXC0HNp1AXfAuNY+rnXD+NY+rnUD7LSuT5hkuyQvm+m+aZ571yR/xnh/z8e19nGtG8a39nGtGzqq3TGBkjRNSVZU1abr+Jy7AvtX1XNnsm+a5z4IWFFV7519hZLWV/YEStIMtT1sxyf5YpKzknw6Sdp9y5O8J8lpSX6c5L5t++FJdh84x4r24buAxyf5eZI3Db3UbfYl2SDJwUl+kuSXSV7TnutNST7RPn5wktOT7AjsC7ypff7j5/a7ImncLOy6AEkaUw8DHgRcBJwMPBY4qd13dVU9OMkewPuBqXry/pY19/bdZl+SZe25H5lkMXByku8AHwCOT7Ib8A7gNVV1RpJDsCdQ0hrYEyhJs/PjqvptVa0Gfg5sN7DvswP/7rIOX/PpwB5Jfg78CNgSuF9bw57Ap4ATqurkdfiaktZT9gRK0uzcNPB4Fbf9fVqTPL6F9g/vJAuADWfxmgH+uqq+Pcm++wErgHvO4rySesieQEla91468O8p7ePlwCPax88HFrWPrwXutIbzDO/7NvDaJIsAktw/ySZJNgc+CDwB2HJg7OFU55bUc4ZASVr37pLkl8AbgInJHocBT0zyC5pLxNe17b8EViX5xSQTQ4b3fQw4A/hZktOBj9L0QP4b8OGqOgd4NfCuJHcDjgZ2c2KIpMm4RIwkrUNJlgNLq2pc72EqqSfsCZQkSeohewIlSZJ6yJ5ASZKkHjIESpIk9ZAhUJIkqYcMgZIkST1kCJQkSeohQ6AkSVIP/X/wNVlDUxgebAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 720x720 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "i=0\n",
        "plot_attention(result['attention'][i], input_text[i], result['text'][i])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e3b6077a",
      "metadata": {
        "id": "e3b6077a",
        "outputId": "e464496a-998c-4ce8-c271-7653dc84e2e8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "she isnt here .\n",
            "the book will run to bed .\n",
            "first dont know her right long whenever me .\n",
            "\n",
            "Wall time: 1.62 s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "three_input_text = tf.constant([\n",
        "     #It might explode.\n",
        "    'Nó có thể nổ.',\n",
        "    #It's payback time.\t\n",
        "    'Đã đến lúc trả đũa rồi.',\n",
        "    # You're not allowed to camp here.'\n",
        "    'Anh không được phép cắm trại ở đây.'\n",
        "])\n",
        "\n",
        "result = translator.tf_translate(three_input_text)\n",
        "\n",
        "for tr in result['text']:\n",
        "  print(tr.numpy().decode())\n",
        "\n",
        "print()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "498dfa29",
      "metadata": {
        "id": "498dfa29",
        "outputId": "b819b7ad-8ec9-40dc-a81c-c6177cf646d3"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(3,), dtype=string, numpy=\n",
              "array([b'she isnt here .', b'the book will run to bed .',\n",
              "       b'first dont know her right long whenever me .'], dtype=object)>"
            ]
          },
          "execution_count": 84,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "result['text']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "16c5b6b7",
      "metadata": {
        "id": "16c5b6b7",
        "outputId": "e53f665b-6ced-4076-ca7c-ce560e2a6558"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\Nhung\\AppData\\Local\\Temp/ipykernel_31496/2069917776.py:14: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
            "  ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
            "C:\\Users\\Nhung\\AppData\\Local\\Temp/ipykernel_31496/2069917776.py:15: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
            "  ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnoAAAI7CAYAAAB7rjbFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAo2ElEQVR4nO3debhkd10n/vcn6SyEEAIBBkUJOxq2AGEJCERwjCKgLOIM+/IzMD+YQRARRJ24QoAZAeUxBJVEVkVgFAUCCgk7YREEEmAghD1sEchOls/8UdWmcunbfbvvUlXfvF7PU09XnXPq1LtO+vZ953u26u4AADCeveYdAACAzaHoAQAMStEDABiUogcAMChFDwBgUIoeAMCgFD2Aqao6vqp+Z945dqSq7lFVn1njskdV1Vc2OxOw+BQ9YK6q6pSq+veq2m/F9LOq6mdmXt+oqrqqtm3Q5z6mqt4zO627n9jdf7AR699o3f3u7r7lRqyrqk6sqj/ciHUBi03RA+amqm6U5B5JOskD5psGYDyKHjBPj0rygSQnJnn09olV9YokN0zypqo6r6qekeRd09nfnU47crrs46rqjOmo4MlVdejMerqqnlhV/7eqvltVL6mJn0xyfJIjp+v67nT5K410VdWvVtXnquqcqvqHqvrRXa175Resqv2r6sKqus709bOr6tKqOmj6+g+q6oXT5/tV1Quq6ktV9Y3pruSrTeddaXdsVd2hqv61qs6tqtdV1d+sHKWrql+vqm9W1der6rHTacckeXiSZ0y/+5um03+zqr46Xd9nquo+a//PCCwqRQ+Yp0cledX0cXRV/ack6e5HJvlSkvt394Hd/bwk95y+5+DptPdX1S8m+a0kD0py3STvTvKaFZ9xvyR3SnLbJA9NcnR3n5HkiUneP13XwSuDVdW9kzxn+p4fSfLFJK/d1bpXrqe7L0ryoST3mk6613Rdd595fer0+XOT3CLJ4UluluQGSX53B9n2TfLGTArytaff+YErFrt+kmtO1/H4JC+pqmt19wmZbO/nTb/7/avqlkmenORO3X2N6fc4a+XnAstH0QPmoqp+KsmhSf62uz+S5PNJHrabq3likud09xndfWmSP05y+OyoXpLndvd3u/tLSd6ZSYlai4cn+avu/mh3X5zkWZmMAN5oD9Z9apJ7TY8vvG2SF09f759JUXzXdDTwmCRP7e5zuvvc6ff5LztY312TbEvy4u6+pLvfkOS0FctckuT3p/PfnOS8JKsd43dZkv2SHFZV+3T3Wd39+dU2DLA8FD1gXh6d5G3d/e3p61dnZvftGh2a5EXTXaffTXJOkspkFGu7s2eeX5DkwDWu+0czGXlLknT3eUm+s4frPjXJUUnukOQTSd6eyUjeXZN8rru/k8mI5AFJPjLzfd46nb6jbF/t7p6Z9uUVy3xnWn53ma+7P5fk15Icm+SbVfXa2d3UwPJS9IAtNz3u7KGZjGqdXVVnJ3lqkttV1e2mi/WKt618nUzKzRO6++CZx9W6+31riLGj9c36WiZFcnvmqyc5JMlX17Duld6XyWjaA5Oc2t2nZ3IM4n1zxW7bbye5MMmtZr7LNbt7R+Xs60lusOKYwB/fjTw/9N27+9XdvX2UtZMctxvrAxaUogfMwy9lsrvwsEx2dx6e5CczOcbuUdNlvpHkJjPv+VaSy1dMOz7Js6rqVklSVdesql9eY4ZvJPmx6fFuO/KaJI+tqsOnl3754yQf7O6z1rj+/9DdFyT5SJIn5Ypi975Mdj2fOl3m8iQvS/InVXW96fe5QVX90HF/Sd6fyfZ7clVtmx6reOfdiHSlbVtVt6yqe0+/50WZFM7Ld2N9wIJS9IB5eHSSl3f3l7r77O2PJH+W5OHTY9mek+S3p7sxnz4tS3+U5L3TaXft7jdmMvL02qr6fpJPJvn5NWZ4R5JPJTm7qr69cmZ3/3OS30ny+kxG0G6aHR8vt1anJtknVxxLd2qSa+SKs4mT5DeTfC7JB6bf55+zg+PquvsHmZyA8vgk303yiCT/mOTiNWb5y0yOx/tuVf2fTI7Pe24mo4pnJ7leJsckAkuurnyIBwDLqKo+mOT47n75vLMAi8OIHsASqqp7VdX1p7tuH53J2bxvnXcuYLFsyK2EANhyt0zyt0munuTMJA/p7q/PNxKwaOy6BQAYlF23AACDUvQAAAal6AEADErRAwAYlKIHADAoRQ8AYFCKHgDAoBQ9AIBBKXoAAINS9AAABqXoAQAMStEDABiUogcAMChFDwBgUIoeAMCgFD0AgEEpegAAg1L0AAAGpegBAAxK0QMAGJSiBwAwKEUPAGBQih4AwKAUPQCAQSl6AACDUvQAAAal6AEADErRAwAYlKIHADAoRQ8AYFCKHgDAoBQ9AIBBKXoAAINS9AAABqXoAQAMStEDABiUogcAMChFDwBgUIoeAMCgFD0AgEEpegAAg1L0AAAGpegBAAxK0QMAGJSiBwAwKEUPAGBQih4AwKAUPQCAQSl6AACDUvQAAAal6AEADErRAwAYlKIHADAoRQ8AYFCKHgDAoBQ9AIBBKXoAAINS9AAABqXoAQAMStEDABiUogcAMChFDwBgUIoeAMCgFD0AgEEpegAAg1L0AAAGpegBAAxK0QMAGJSiBwAwKEUPAGBQih4AwKAUPQCAQSl6AACD2jbvAKyuqh60B297S3dfuOFhAIClU9097wysoqou3823dJKbd/eZm5EHAFguRvQW3/W7+5trWbCqzt3sMADA8nCM3mI7Kcnu7IZ9ZZLvb1IWAGDJ2HULADAoI3oLrqouq6rrzTsHALB8FL3FV/MOAAAsJ0UPAGBQzrpdDg+tqp2eZNHdf71VYQCA5eBkjAU3vZbeBZlcI2813d0HbVEkAGBJKHoLblr01nwtPQCA7Ryjt/g0cQBgjyh6i89ZtwDAHnEyxuLb6d0xquqIJH/Y3T+3dZEAYDxVde3dfU93n7MZWTaKY/SWQFX95yQ/m+SSJH/R3WdW1S2SPD/J/ZK8XdEDgPWZHhe/O8Wok9yiu8/cpEjrZkRvwVXVo5O8PMk5Sa6d5PFV9ZQkL03yhiSHd/cn5hgRAEbykEx+5+5KJXnzJmdZNyN6C66qPpbktd393Kp6aJLXJvnXJA/t7s/PNRwADKSqvpDkiO7+zhqX/2SSn+/uL29usj2n6C24qjo3yW27+wtVtVeSi5P8THefOudoAMCCc9bt4rt6kvOTpLsvT3JRkoX9PwcAYHE4Rm85/EJVfW/6fK8kR1fVN2YX6O43bH0sABhPVVWSRyV5cJKbZHLSxZlJXpfkVb1Eu0Ptul1w0zOAdqW7e+9NDwMAVwFV9YYkv5TkE0lOz+TEi8OS3DrJG7v7wfNLt3uM6C247rZ7HQC2SFU9PJNLmv1cd79txbyjk7y+qh7W3a+eS8DdpEQMoKp+Zt4ZAGAQj0hy3MqSlyTdfXIm17B9xJan2kOK3pKqqhtU1W9X1ZlJTp53HgAYxO2y8+vj/VOSw7cmyvopekukqvauqgdV1T8lOSvJA5Mcn+Rmcw0GAOM4JMnXdzL/65ncwGApOEZvCVTVLZP8f5mcAXR+kldncvzAI7v79HlmA4DB7JPJLUdXc+l0maWg6C24qnp3Jmf5vD6Tu2GcOp3+m3MNBgDjek5VXbDKvAO2NMk6KXqL78gkL0lyQnd/at5hAGBw70py0zUssxQUvcV3p0x2276nqs5K8tdJXjPXRAAwqO4+at4ZNpILJi+Jqto/yS8neVySn8rkRJpnJvmL7v73eWYDABaTorfgquqGSb48e7uVqrpZrjg545Ak7+jun59TRGAXquo6mewK+lh3XzzvPMDqquppa1muu//3ZmfZCIregquqy5L8SHd/cwfz9k5yvySP6+5f3PJwwE5V1TWS/GWSh2Ryr8ybd/eZVXV8krO7+9h55gN+WFV9YSezO8n1k+y3LLcedR29xVerzejuy7r775U8WFjHJblBkjskuXBm+j9mch1MYMF094139EhynyQfTLJvktfNN+XaKXoAm+cBSX6tuz+WyUjAdmckuclcEgG7paoOqaoXJjk9yfWS3LW7/8t8U62ds26Xw9Or6rydLdDdv79VYYA1u1aS7+xg+jWSXLbFWYDdUFVXS/K0JM/I9G5U3f2WuYbaA4recrh/JlfiXk0nUfRg8Xwok1G9F05fbx/Ve0KS980jELBzVbVXkscn+b1M7pDx35O8opf0pAZFbznca0cnYwAL77eSnFxVt8rk39unTZ/fOck955oMWM0nkxya5MVJ/jTJRUmuVXXlQ+a7+5ytj7b7nHW74HZ21i1rV1X7JXl4ksMyGVX5VJLXuNTF2lTVTXLFtjuju8+cc6SlUVW3SfL0JHfM5LjojyY5rrs/MddgS8LPLlutqi6febmjklRJelnOulX0Ftz0L9z1Fb09V1WHJXlLkmsm2f7L9TZJvpfk57r7jHllW3RVdVAmlwd5cJLt//hVJvdefnx3nzuvbIzPzy7zUFX3Wsty2+89v+gUvQVXVf8zyfO7e7WbK7MLVfX2JBckeWR3f3867aAkr8zkWkhHzzPfIquqlye5W5JjcsUxZXdPcnyS93b34+eVbZlU1Y9mcrbela500N0fnU+i5eBnF9ZP0VtwVXVgJv+gfWdm2k8m+Y0kByZ5Q3e/dl75lkFVXZDkTt39qRXTb5PkA9199fkkW3xV9Z0kv9Td714x/Z5J3tjdh8wn2XKoqttnUkp+Ij98Tcyl2fUzL352mYeqOibJSdsPD5geV/uZ7r50+vrqSX6zu393jjHXzHX0Ft+fZ3LmT5L/uJXSuzO5I8Ytk7yqqh42p2zL4qIkB+9g+jWn81jd1bLjy4Ock2T/Lc6yjE5I8uUk98jkunk3nnm4jt6u+dllHv48k79j270/yQ1nXh+Y5NlbmmgdFL3Fd2SSN868fmSSH2RyK6XbJXlBkifPI9gSeVOSl1XV3atq7+njp5K8NMk/zDnbontvkj+oqgO2T5j+3+zvxeVB1uKwJP+ju9/X3Wd19xdnH/MOtwT87DIPK0ffV71D1TJQ9BbfjyT5/Mzrn07y+u7+3vT1SUluvuWplstTkvzfTEZCL5o+3pXks0meOsdcy+BpSe6a5KtVdWpVnZrJCNVdk/zaPIMtiU9kcl9M9oyfXVgn19FbfBckmT0O5c5J/mbm9UVJDgir6u7vJvnFqrpZkp+cTj6juz83v1TLobs/UVU3z+TyFj8xnfyKJK/q7gtXf+dVV1Vde+blbyV5XlX9dial75LZZZflOlzz4mcX1k/RW3wfT/LYTG6DdlSS6yZ5x8z8myb52tbHWi5V9SuZ3JD6P8583H7xy+5+wPySLbaq+qMkX+7u41dMf2JV3aC7f2dO0RbZt3Pla29VkrftYFoncTLGLvjZZU5+oaq27znbK8nRVfWN6euD5xNpzyh6i+8Pkrylqh6aSck7sbu/PjP/gUneM5dkS6Kqnp/JbsZ3ZlKKnWq+do9M8ss7mP7RJM9Kouj9sJ+eeX6jTHZ1r7yv7V658sHd7ICfXeboL1e8fsmK10vzd9HlVZbA9HIqP5vk7CSv6+7LZ+Ydk+S07v7YnOItvOn/hT2pu/9u3lmWTVVdlOSwlXfCmN4p4/TudubtTqx2Z5uqOiTJN11eZef87ML6ORljgVXVnatq7+4+o7tf1N1/M1vykqS7T9he8qrqjlW1z1zCLra9knxs3iGW1JcyuTTISvdM8pUtzrKMtu+iXenAuDzIWvjZZUtt/727G8sv/O9du24X2/szOWPvW2tc/p1JDk/iPqRXdkKSRyQ5ds45ltFLk/xJVe2bK44NvU+S5yQ5bm6pFlxVvXj6tJM8Z3rh3+32zuSkqo9tda4l5GeXrTbc711Fb7FVfviXxM7su5lhltjBSR5WVf85yb/lh898/B/zCLUMuvt/TS/S/eJc8ffrB0le1N3Pm1+yhXeb6Z+VydmiP5iZ94NMjnF8wVaHWkIHx8/uhquqMzK5FqsO8MOG+73rGL0FVlWnZPcP+HzYipM1rvKq6p07md3dfe8tC7OkphdJPmz68ozuPm+eeZbF9F7BT9l+n1Z2j5/dzVFVT05ySHf/3i4XvooZ8feuogcAMCgnYwAADErRW0LTS6qwh2y/9bH99pxttz623/rYfuuzrNtP0VtOS/mXbYHYfutj++052259bL/1sf3WZym3n6IHADAoJ2PswL61X+9fV593jFVd0hdnn9pv3jFWdYvbrPWs9Pn41ncuy3UPWcwbEnz6S9edd4RduuTi87LPfgfOO8YOXbbg9+m49ILzs+2Axf23ZZ9zL9/1QnN0yaXnZ59tC7r9zr9w3gl26ZJcnH2yuL87Ft0ib79z8+/f7u4d/gJxDZ0d2L+unrtuO3reMZbWW0/+8LwjLK17PPkJ846w1M75icUs8MviBqecP+8IS6s+8Ml5R1hul6+8HTS745/777642jy7bgEABqXoAQAMStEDABiUogcAMChFDwBgUIoeAMCgFD0AgEEpegAAg1L0AAAGpegBAAxK0QMAGJSiBwAwKEUPAGBQih4AwKAUPQCAQSl6AACDUvQAAAal6AEADErRAwAYlKIHADAoRQ8AYFCKHgDAoBQ9AIBBKXoAAINS9AAABqXoAQAMStEDABiUogcAMChFDwBgUIoeAMCgFD0AgEEpegAAg1L0AAAGtZRFr6oeU1XnzTsHAMAiW8qiBwDAril6AACDWuiiV1X3rKoPVNV5VfW9qjqtqm49M/8+VfXJqjq/qt5ZVTde8f77V9VHquqiqvpCVf1RVe279d8EAGDrLWzRq6ptSf4+yXuS3C7JXZK8MMll00X2S/KsJI9LcmSSg5McP/P+o5O8KsmfJbnVdLmHJPnjrcgPADBv2+YdYCcOyqS8vam7Pz+d9ukkqaq7ZJL9Sd39mem0FyT5q6qq7u4kz07y/O5++fS9n6+q30zyyqr6jeky/6GqjklyTJLsnwM295sBAGyBhR3R6+5zkpyY5OSq+qeqelpV3XBmkYu3l7ypryXZN8m1pq/vmOTZ092+503P0n11kqsnuf4OPu+E7j6iu4/Yp/bbjK8EALClFrboJUl3PzaTXbbvSvKAJJ+Z7pJNkktXLj79c6+ZP38vyeEzj9smuXmSb21WZgCARbHIu26TJN398SQfT3JcVb0lyaOTvG0Nb/1okp/o7s9tZj4AgEW1sEVvegbtE5L8Q5KvJrlJJiNyf77GVfx+kn+sqi8m+dtMRgBvneTO3f2MjU8MALBYFnnX7QVJbpHkdUk+m+SkTM6iPW4tb+7uk5P8QpKfTnLa9PHMJF/ajLAAAItmYUf0uvsbSR60yuwTp4/Z5U9JUiumvS1r280LADCcRR7RAwBgHRQ9AIBBKXoAAINS9AAABqXoAQAMStEDABiUogcAMChFDwBgUIoeAMCgFD0AgEEpegAAg1L0AAAGpegBAAxK0QMAGJSiBwAwKEUPAGBQih4AwKAUPQCAQSl6AACDUvQAAAal6AEADErRAwAYlKIHADAoRQ8AYFCKHgDAoBQ9AIBBKXoAAINS9AAABqXoAQAMStEDABiUogcAMChFDwBgUIoeAMCgts07wELqpC+7bN4pltbP3+xu846wtK52+4vnHWGpveZ/Hz/vCEvtIbc9Zt4RltbNz/7xeUdYapd99evzjrDcLlp9lhE9AIBBKXoAAINS9AAABqXoAQAMStEDABiUogcAMChFDwBgUIoeAMCgFD0AgEEpegAAg1L0AAAGpegBAAxK0QMAGJSiBwAwKEUPAGBQih4AwKAUPQCAQSl6AACDUvQAAAal6AEADErRAwAYlKIHADAoRQ8AYFCKHgDAoBQ9AIBBKXoAAINS9AAABqXoAQAMStEDABiUogcAMChFDwBgUIoeAMCgFD0AgEEpegAAg1rYoldVJ1bVP847BwDAsto27wA78ZQktRErqqpjkzyku2+9EesDAFgGC1v0uvt7884AALDMlmLXbVXds6o+UFXnVdX3quq0qrr1dN5jptPvU1WfrKrzq+qdVXXj7fOT/M8kt6qqnj4eM6/vBQCwVRZ2RG+7qtqW5O+T/GWShyfZJ8kdklw2s9h+SZ6V5HFJLkpyUpLjkxyd5G+S3DrJ/ZIcNV3eaCEAMLyFL3pJDkpycJI3dffnp9M+vWKZbUme1N2fSZKqekGSv6qq6u4Lq+q8JJd299mrfUhVHZPkmCTZPwds8FcAANh6C7vrdrvuPifJiUlOrqp/qqqnVdUNVyx28faSN/W1JPsmudZufM4J3X1Edx+xT/Zbd24AgHlb+KKXJN392CR3SfKuJA9I8pmqOnpmkUtXvmX651J8PwCAzbA0Rai7P97dx3X3UUlOSfLo3Xj7D5LsvRm5AAAW1cIXvaq6cVU9t6ruVlWHVtVPJ7ltktN3YzVnJTm0qu5QVdepKvtmAYDhLXzRS3JBklskeV2Sz2ZyRu2rkhy3G+t4fZI3J/mXJN9K8l83OCMAwMJZ2LNuu/sxMy8ftJPlTszkZI3Zaadk5q4a3X1xkodsZD4AgEW3DCN6AADsAUUPAGBQih4AwKAUPQCAQSl6AACDUvQAAAal6AEADErRAwAYlKIHADAoRQ8AYFCKHgDAoBQ9AIBBKXoAAINS9AAABqXoAQAMStEDABiUogcAMChFDwBgUIoeAMCgFD0AgEEpegAAg1L0AAAGpegBAAxK0QMAGJSiBwAwKEUPAGBQih4AwKAUPQCAQSl6AACDUvQAAAal6AEADErRAwAY1LZ5B1hIVal99513iqV1+UUXzzvC0trrw2fMO8JSe+ad7j/vCEvtx2+7z7wjLK2TTn3VvCMstbuc8uR5R1huj1h9lhE9AIBBKXoAAINS9AAABqXoAQAMStEDABiUogcAMChFDwBgUIoeAMCgFD0AgEEpegAAg1L0AAAGpegBAAxql0WvqvZbyzQAABbLWkb03r/GaQAALJBtq82oqusnuUGSq1XV7ZPUdNZBSQ7YgmwAAKzDqkUvydFJHpPkx5L8r1xR9L6f5Lc2NxYAAOu1atHr7pOSnFRVD+7u129hJgAANsBajtH7paq65vYXVXVoVf3LJmYCAGADrKXovSfJB6vqvlX1q0nenuSFm5oKAIB129kxekmS7n5pVX0qyTuTfDvJ7bv77E1PBgDAuqzlOnqPTPJXSR6V5MQkb66q221yLgAA1mmXI3pJHpzkp7r7m0leU1VvTHJSksM3MxgAAOuzll23v5QkVXVAd1/Q3adV1Z03PRkAAOuyll23R1bV6Uk+PX19uzgZAwBg4a3lrNsXZnLx5O8kSXd/PMk9NzETAAAbYC1FL9395RWTLtuELAAAbKC1nIzx5aq6W5Kuqn2SPCXJGZsbCwCA9VrLiN4TkzwpyQ2SfDWTs23//03MBADABljLiN4tu/vhsxOq6u5J3rs5kQAA2AhrGdH70zVOAwBggaw6oldVRya5W5LrVtXTZmYdlGTvzQ4GAMD67GzX7b5JDpwuc42Z6d9P8pDNDAUAwPqtWvS6+9Qkp1bVid39xS3MBADABtjlMXpKHgDAclrTBZM3U1WdUlV/Nu8cAACjWcu9bu++lmkAACyWIS+vMr2DBwDAVdqqRa+qjqyqX8/08iozj2Oz8ZdX2auq/riqvl1V36yqF1TVXtMc+1bVcVX1laq6oKo+VFVHz+Q8qqq6qu5bVadV1Q+SHF0Tz6iqz1fVhVX1iap6xAbnBgBYWItyeZWHJ3lRJtftOzzJq5N8JMlrkrw8yU2TPCzJV5LcN8mbqupO3f3xmXUcl+TXk3wuyblJ/nCa80lJPpPkyCQvq6p/7+5/2uD8AAALZ1Eur3J6d//u9Plnq+pXk9ynqk5L8l+T3Ki7vzSd/2dV9TNJnpAr33P32O5+W5JU1dWTPC3Jz3b3u6fzv1BVd86k+Cl6AMDw1nKv2xOrqldO7O57b2COf1vx+mtJrpfkDkkqyelVNTt/vyTvWPGeD888PyzJ/kneuiL7PknO2lGAqjomyTFJsn8O2L30AAALaC1F7+kzz/dP8uAkl25wjktWvO5Mjh/ca/r8TjtY5sIVr8+feb792MP7J/nSiuVWrmfygd0nJDkhSQ7a65AfKrYAAMtml0Wvuz+yYtJ7p7tUt8K/ZjKid/3ufuduvO/0JBcnObS7V478AQBcJeyy6FXVtWde7pXkjkmuuWmJZnT3Z6vqVZnsPv71JB9Ncu0kRyU5s7vfsMr7zq2qFyR5QU32+b4rkxNL7prk8unoHQDA0Nay6/Yjmew+rUx22X4hyeM3M9QKj03y7CTPS/JjSc5JclqSXY3w/U6Sb2Sy6/nPMzlb+GPT9QAADG8tu25vvJkBuvuoHUx7zMzzS5IcO33s6P2nZFJCV07vTC7svNAXdwYA2Cxr2XW7fyaXMfmpTEb23p3k+O6+aJOzAQCwDmvZdfvXmVyAePvI2MOSvCLJL29WKAAA1m8tRe/W3X3YzOt3VtXpmxUIAICNseq9bmd8tKruuv1FVd0lV744MQAAC2gtI3p3TPK+qtp+4eEbJvlMVX0ik3Mebrtp6QAA2GNrKXo/t+kpAADYcGspen/Y3Y+cnVBVr1g5DQCAxbKWY/RuNfuiqrZlsjsXAIAFtmrRq6pnVdW5SW5bVd+vqnOnr7+R5O+3LCEAAHtk1aLX3c/p7mskeX53H9Td15g+DunuZ21hRgAA9sBajtF7S1Xdc+XE7n7XJuQBAGCDrKXo/cbM8/2T3DnJR5Lce1MSAQCwIXZZ9Lr7/rOvq+rHk7xwswIBALAx1nLW7UpfSfKTGx0EAICNtcsRvar60yQ9fblXksOTfHQTMwEAsAHWcoze7H1tL03ymu5+7yblAQBgg6yl6P1NkptNn3+uuy/axDwAAGyQnV0weVtVPS+TY/JOSvLXSb5cVc+rqn22KiAAAHtmZydjPD/JtZPcuLvv2N13SHLTJAcnecEWZAMAYB12VvTul+RXu/vc7RO6+/tJ/luS+252MAAA1mdnRa+7u3cw8bJccRYuAAALamdF7/SqetTKiVX1iCSf3rxIAABshJ2ddfukJG+oqsdlcsuzJDkiydWSPHCzgwEAsD6rFr3u/mqSu1TVvZPcajr5zd39L1uSDACAdVnLvW7fkeQdW5AFAIANtCf3ugUAYAkoegAAg1L0AAAGpegBAAxK0QMAGJSiBwAwKEUPAGBQih4AwKB2ecHkq6Lae+/sdfA15x1jaV3+3e/NOwJXVZdeOu8ES23fb18w7whL69iz7z3vCEvtUbf94LwjLLXf38k8I3oAAINS9AAABqXoAQAMStEDABiUogcAMChFDwBgUIoeAMCgFD0AgEEpegAAg1L0AAAGpegBAAxK0QMAGJSiBwAwKEUPAGBQih4AwKAUPQCAQSl6AACDUvQAAAal6AEADErRAwAYlKIHADAoRQ8AYFCKHgDAoBQ9AIBBKXoAAINS9AAABqXoAQAMStEDABiUogcAMChFDwBgUIoeAMCgFD0AgEEpegAAg1L0AAAGpegBAAxK0QMAGJSiBwAwKEUPAGBQih4AwKC2zTvAoqiqY5IckyT773XgnNMAAKyfEb2p7j6hu4/o7iP23etq844DALBuih4AwKAUPQCAQV2lil5VPbmqPj3vHAAAW+EqVfSSXCfJLecdAgBgK1ylil53H9vdNe8cAABb4SpV9AAArkoUPQCAQSl6AACDUvQAAAal6AEADErRAwAYlKIHADAoRQ8AYFCKHgDAoBQ9AIBBKXoAAINS9AAABqXoAQAMStEDABiUogcAMChFDwBgUIoeAMCgFD0AgEEpegAAg1L0AAAGpegBAAxK0QMAGJSiBwAwKEUPAGBQih4AwKAUPQCAQSl6AACDUvQAAAal6AEADErRAwAYlKIHADAoRQ8AYFCKHgDAoLbNO8BC2nuv5KAD551iaV1+sx+dd4SlVZdcNu8IS+3Ca+8/7whL7awHzzvB8vr82+8w7whL7dC3XDTvCEvuH1adY0QPAGBQih4AwKAUPQCAQSl6AACDUvQAAAal6AEADErRAwAYlKIHADAoRQ8AYFCKHgDAoBQ9AIBBKXoAAINS9AAABqXoAQAMStEDABiUogcAMChFDwBgUIoeAMCgFD0AgEEpegAAg1L0AAAGpegBAAxK0QMAGJSiBwAwKEUPAGBQih4AwKAUPQCAQSl6AACDUvQAAAal6AEADErRAwAYlKIHADAoRQ8AYFCKHgDAoDat6FXVKVXV08ddN+tz1pjlrJks15lnFgCArbLZI3ovT/IjST6SJDNla+XjidP5R01ff7qqts2uaFrWnj7zerZI/qCqvl5Vb62qR1RVrchxpyQP3tyvCgCwWDa76F3Q3Wd39yUz0341k/I3+zhpxfsOTfL4Nax/e5G8SZIHJHl/kpcmeWNV7b19oe7+VpJz9vRLAAAso227XmTDfbe7z97FMi9OcmxVvbK7z9/JchfMrOsrST5UVR9I8tYkj8qkCAIAXCUt6skYf5rkkiRP2903dvfJST4Ru2oBgKu4eRS9V1TVeSset1mxzEVJfifJb1TVdffgM07PZHfumlXVMVX14ar68A8uu3APPhIAYLHMo+j9RpLDVzw+s4PlXpHkrEwK3+6qJL07b+juE7r7iO4+Yt+9r7YHHwkAsFjmcYze2d39uV0t1N2XV9Uzk/yfqnrRbn7GYUnO3KN0AACDWNRj9JIk3f3mJO9N8kdrfU9VHZ3k1kn+brNyAQAsg3mM6B1cVddfMe287j5vleWfkeQDmZycsdIB03Vty+QyK/edLv/3SV65QXkBAJbSPEb0Xpbk6ysez1xt4e7+UCajc/vtYPZjp+8/M8mbkhyZ5IlJHtjdl21sbACA5bKlI3rdvfKOFSvnn5LJiRQrp/9Kkl9ZMe2ojcwGADCazR7RO2Z6+ZQ7bfLn7FRVfSrJW+aZAQBgq23miN7Dk2y/TsmXN/Fz1uK+SfaZPncrNADgKmHTil53f3Wz1r27uvuL884AALDVFvryKgAA7DlFDwBgUIoeAMCgFD0AgEEpegAAg1L0AAAGpegBAAxK0QMAGJSiBwAwKEUPAGBQih4AwKAUPQCAQSl6AACDUvQAAAal6AEADErRAwAYlKIHADAoRQ8AYFCKHgDAoBQ9AIBBKXoAAINS9AAABqXoAQAMStEDABiUogcAMChFDwBgUIoeAMCgFD0AgEEpegAAg1L0AAAGVd097wwLp6q+leSL886xE9dJ8u15h1hitt/62H57zrZbH9tvfWy/9Vnk7Xdod193RzMUvSVUVR/u7iPmnWNZ2X7rY/vtOdtufWy/9bH91mdZt59dtwAAg1L0AAAGpegtpxPmHWDJ2X7rM/T2q6rzNmGdN6qqh2UH225m3p6u+6iqutu6Ai6Pof/ubQHbb32Wcvs5Rg9gRlWd190HbvA6j0ry9O6+3+7MW+O6j01yXne/YM8TAqMyogewA9ORslOq6u+q6tNV9aqqqum8s6rqeVX1iao6rapuNp1+YlU9ZGYd20cHn5vkHlX1sap66oqPutK8qtq7qp5fVR+qqn+rqidM1/XUqvqr6fPbVNUnq+qwJE9M8tTp+++xuVsFWDbb5h0AYIHdPsmtknwtyXuT3D3Je6bzvtfdt6mqRyV5YZKdjcg9M6uP2l1pXlUdM133napqvyTvraq3JXlRklOq6oFJnp3kCd19elUdHyN6wCqM6AGs7rTu/kp3X57kY0luNDPvNTN/HrmBn/mzSR5VVR9L8sEkhyS5+TTDY5K8Ismp3f3eDfxMYFBG9ABWd/HM88ty5X8zewfPL830f6Craq8k++7BZ1aS/97dJ+9g3s2TnJfkR/dgvcBVkBE9gD3zKzN/vn/6/Kwkd5w+f0CSfabPz01yjVXWs3LeyUn+W1XtkyRVdYuqunpVXTPJi5PcM8khM8cC7mzdwFWcogewZ65VVf+W5ClJtp9g8bIk96qqj2eyO/f86fR/S3JZVX18BydjrJz3F0lOT/LRqvpkkpdmMpL4J0le0t2fTfL4JM+tqusleVOSBzoZA9gRl1cB2E1VdVaSI7p7Ue97CZDEiB4AwLCM6AEADMqIHgDAoBQ9AIBBKXoAAINS9AAABqXoAQAMStEDABjU/wNo8KlM7ntmswAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 720x720 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "i = 0\n",
        "plot_attention(result['attention'][i], three_input_text[i], result['text'][i])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c13f7f8f",
      "metadata": {
        "id": "c13f7f8f",
        "outputId": "16dd3fdc-4b69-480a-f6df-099e93e2d98d"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\Nhung\\AppData\\Local\\Temp/ipykernel_31496/2069917776.py:14: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
            "  ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
            "C:\\Users\\Nhung\\AppData\\Local\\Temp/ipykernel_31496/2069917776.py:15: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
            "  ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnoAAAJsCAYAAABqJQ6KAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAwbklEQVR4nO3debxdZX3v8c83hBAGAQUtSqs4UkQENSrUAWq9zrVO1VacvSKtVuuMrdzaWut8HdsidUBxaOtU9Tq21aBQRMG5gFYxggIKqIxiAvndP9aK2WxOkpPk7L32ec7n/Xqd19lr3L+V7OR8z7Oe51mpKiRJktSeZUMXIEmSpMkw6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSb0kxyU5dug65pLkXkm+M899j0jyo0nXJGn2GfQkDSrJ6iQ/T7LT2Po1Se47srxfkkqyfIHe90lJTh5dV1VHV9XLFuL8C62qvlhV+y/EuZKckORvF+JckmabQU/SYJLsB9wLKOChw1YjSe0x6Eka0hOALwEnAE/csDLJicDNgY8nuSLJC4Ev9Jt/0a87rN/3KUnO6lsFP5PkFiPnqSRHJ/mfJL9I8vfpHAAcBxzWn+sX/f7XaelK8rQk30vysyQfS3KzLZ17/AKTrEzyyyR798t/meSaJLv3yy9L8ob+9U5JXpvk3CQ/6W8l79xvu87t2CR3TvK1JJcn+UCSfxlvpUvyvCQ/TXJBkif3644CjgRe2F/7x/v1L0ry4/5830nye/P/a5Q0qwx6kob0BOC9/df9k/wGQFU9HjgX+P2q2q2qXg3cuz9mz37dqUn+APgL4BHAjYEvAu8fe4+HAHcF7gg8Grh/VZ0FHA2c2p9rz/HCktwHeEV/zE2BHwL/vKVzj5+nqq4GvgIc3q86vD/XPUaWT+pfvxK4HXAIcBtgX+D/zFHbCuAjdAH5Rv01P3xst32APfpzPBX4+yQ3rKrj6f68X91f++8n2R94JnDXqrpBfx1rxt9X0uJj0JM0iCT3BG4B/GtVnQF8H3jsVp7maOAVVXVWVV0D/B1wyGirHvDKqvpFVZ0LfJ4uRM3HkcA7quqrVfUr4MV0LYD7bcO5TwIO7/sX3hF4U7+8ki4ofqFvDTwKeE5V/ayqLu+v54/mON+hwHLgTVW1rqo+DHx5bJ91wN/02z8JXAFsqo/ftcBOwO2T7FhVa6rq+5v6g5G0eBj0JA3licBnq+rifvl9jNy+nadbAG/sb53+AvgZELpWrA0uHHl9FbDbPM99M7qWNwCq6grgkm0890nAEcCdgW8B/07Xknco8L2quoSuRXIX4IyR6/l0v36u2n5cVTWy7ryxfS7pw+8W66uq7wF/DrwU+GmSfx69TS1p8TLoSZq6vt/Zo+latS5MciHwHODgJAf3u9XYYePL0IWbp1fVniNfO1fVf82jjLnON+p8uiC5oeZdgb2AH8/j3OP+i6417eHASVV1Jl0fxAex8bbtxcAvgQNHrmWPqpornF0A7DvWJ/C3tqKe6117Vb2vqja0shbwqq04n6QZZdCTNISH0d0uvD3d7c5DgAPo+tg9od/nJ8CtRo65CFg/tu444MVJDgRIskeSP5xnDT8BfrPv7zaX9wNPTnJIP/XL3wGnVdWaeZ7/16rqKuAM4BlsDHb/RXfr+aR+n/XAPwGvT3KT/nr2TXK9fn/AqXR/fs9Msrzvq3i3rSjpOn+2SfZPcp/+Oq+mC5zrt+J8kmaUQU/SEJ4IvLOqzq2qCzd8AW8Bjuz7sr0CeEl/G/P5fVh6OXBKv+7QqvoIXcvTPye5DPg28MB51vA54L+BC5NcPL6xqv4DOBb4EF0L2q2Zu7/cfJ0E7MjGvnQnATdg42higBcB3wO+1F/PfzBHv7qqWks3AOWpwC+AxwH/D/jVPGt5O11/vF8k+Te6/nmvpGtVvBC4CV2fREmLXK7bxUOStBglOQ04rqreOXQtkmaHLXqStAglOTzJPv2t2yfSjeb99NB1SZotC/IoIUnS1O0P/CuwK3AO8KiqumDYkiTNGm/dSpIkNcpbt5IkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqOWD13AUpXkEdtw2Keq6pcLXowkSWpSqmroGpakJOu38pACbltV50yiHkmS1B5b9Ia1T1X9dD47Jrl80sVIkqS22EdvOO8CtuY27HuAyyZUiyRJapC3biVJkhpli96Aklyb5CZD1yFJktpk0BtWhi5AkiS1y6AnSZLUKEfdDu/RSTY7yKKq3j2tYiRJUjscjDGgfi69q+jmyNuUqqrdp1SSJElqiEFvQH3Qm/dcepIkSVvDPnrDMmVLkqSJMegNy1G3kiRpYhyMMazNPh0jySrgb6vqAdMrSZIkbUqSG23tMVX1s0nUMh/20RtYkv8F3A9YB7ytqs5JcjvgNcBDgH836EmSNBv6/vVbE54KuF1VnTOhkjbLFr0BJXki8E7gZ8CNgKcmeTbwVuDDwCFV9a0BS5QkSdf3KLqf3VsS4JMTrmXzBdiiN5wkXwf+uapemeTRwD8DXwMeXVXfH7Q4SZJ0PUl+AKyqqkvmuf+3gQdW1XmTrWwT72/QG06Sy4E7VtUPkiwDfgXct6pOGrg0SZLUAEfdDmtX4EqAqloPXA0MkvglSVJ77KM3vAcnubR/vQy4f5KfjO5QVR+eflmSJGlTkgR4AvBI4FZ0gy7OAT4AvLdm5Japt24H1I/c2ZKqqh0mXowkSZq3JB8GHgZ8CziTbuDF7YE7AB+pqkcOV91GtugNqKq8dS5J0iKT5Ei6qdEeUFWfHdt2f+BDSR5bVe8bpMARBo0Zl+S+Q9cgSZKu43HAq8ZDHkBVfYZuLtzHTb2qORj0ZlCSfZO8JMk5wGeGrkeSJF3HwWx+frxPAIdMp5TNM+jNiCQ7JHlEkk8Aa4CHA8cBtxm0MEmSNG4v4ILNbL+A7kEIg7OP3sCS7A/8b7qRO1cC76O77//4qjpzyNokSdKcdqR7dOmmXNPvMziD3oCSfJFudM6H6J6GcVK//kWDFiZJkrbkFUmu2sS2XaZayWYY9IZ1GPD3wPFV9d9DF6PpSXIz4CaMdZ+oqq8OU5G08PrP+c2BFaPrq+oLw1QkLZgvALeexz6DM+gN6650t21PTrIGeDfw/kEr0kQluRPwHuC36eZcGlWAcyZq0esD3vuAe9N9rtN/38DPuRa1qjpi6Brmy8EYA6qqr1XVM4CbAv8XeCjdI9CW0T0x44ZD1qeJOJ7u7/hedDOp33Lk61YD1iUtpDcA19JNHnsV3ef9D4GzgAcMV5a09PhkjAEluTlw3uhjUpLcho2DM/YCPldVDxyoRC2wJFcCd6qq7w5dizQp/WMcH1xVpye5DFhVVd9N8mDg2Ko6dOASpe2S5Lnz2a+q/u+ka9kSg96AklwL3LSqfjrHth2AhwBPqao/mHpxmogkXwJeaB8ltawPd3esqjV9t5THVdXJSW4J/HdVzUxHdWlbJPnBZjYXsA+w0yw8wtQ+esMa76P1a1V1LfDR/kvt+Avg1UleQvd8xOsMz6+qnw1SlbSwzqbrh7oG+DpwdJLzgGcAPx6uLGlhVNUt51qf5FbAy+m6KnxgqkVtgi16A0qyHthnrhY9tan/O99g9B9fgJqF3/6k7dU/B3THqjohyZ2BT9N1RfkV8MSqmokfgNJCSbIXcCxwNHAK8KKqOn3YqjoGvQH1P/RfC1yxuf2q6m+mU5EmLcnhm9u+YS7FFiV5OV2f1OPG1h8N7FtVxw5T2WQluR3wKOaeZuQpgxQ1ZUl2oWvhO7eqLh66HmmhJNkZeC7wQroW7GOq6lODFjXGoDegPuh9h24G7U2pqrrjlEqSJibJucAfVtVpY+vvCnywqm4xTGWT0w8++BDwNeAuwFfo5t7aCfhiVT10wPIkbaMky4CnAn9N1wXnWODEmsFQZR+94R3urdulJclBwNPpfuA/paouSPIw4IdV9bVBi5usmwAXzbH+EuA3plzLtPwN8NdV9YoklwOPB84HTgROHbSyCUryps1tr6pnTasWaUK+DdwCeBPwZuBq4IbJdbvez0K/a4PesGYu+WuyktwP+BjwKeA+wM79plsDTwIeNkhh03Eu3Xxq54ytvzfwo+mXMxX7A//Sv14H7FJVVyf5G+ATdPNntuigseUd6W7d7kDXutmUJB+jG1l8Wf96k2zFbcZv999fRHfbdtyGScIH73dt0BvWJkfdqlkvA55bVf/Qt/BssBp43jAlTc1bgdcnWQF8rl/3e8ArgFcNVtVkXQ6s7F9fANyGriVgOdDshOhV9bvj65KsBN4OfHH6FU3cJWz8xf2SIQvR1FzvMz6rDHrD+mu2MBBDzbkD8Mk51v8MuNGUa5mqqnpdkr3pbnVsGJSwFnhjVb16uMom6jTgnsCZdC14r0tyMPBwGr51O5e+JfPv6EbgHrel/ReTqnryXK/VrsU0cM5HoA3rdWy8dQdAkgOSvCPJvyb5o4Hq0uT8DNh3jvV3pt3bl79WVS8G9gYO7b9uXFXHDFvVRD0X+FL/+qXAZ4FHAt+jewLOUrM3sNvQRUxDkpVJ7pDkwL41Uw1JclSSnUaWD0yyfGR5176LxuAcdTugJCcCl1bVM/vlvekmGl1Pd5vnDsDjq+p9w1WphZTkVXT91B5N18qziu5ZxycA73QqnXb0/+nfDzitqpbU7bw5Hg8Vus/5kXSPdTxy+lVNR5Idgb8DnknXch26+QPfDPxlVa3bzOFaJMafbNU/DeaQqjqnX/4N4PxZmBvVW7fDOoxu9OUGj6e7lXVAVV3ah4JnAga9dryELtT9kO4HwJl0LevvpZtNvVlLrZN6VV2T5MN0nbaXVNAD/mxseT3diOt30vXJbNmrgD+mmzj35H7dveiuexnw/IHq0sIa72M/s33uDXrDuinw/ZHl3wU+VFWX9svvApbEhKqtS/Iu4AX9b39HJjmW7nbtMuBrVfU/gxY4HeNhZ0fgYOC3gA9Pv5yp+AbdAIw1A9cxFSOf8zkfD7VEPJZu2qTRvrjfT3IR8DYMepoyg96wrgJ2HVm+GxunYoBuXp5mH/7d39q6G3M/MeDdgxQ1AUneCfwRsMvYSNsNHrBh7qWWn5SwqU7qSV4HXDblcqblpXQDMP4KOAO4cnTjLMyxtVDm8Tn/tZY/58AeXPcX+A2+D+w53VIkg97QvgE8GXh+kiOAG7Nx2gno5lY7f/plTV6S3wY+DtySrsn7WrrP4zq6/izNBD26v9fQjapdSTdv3HrgW/32O9C17H1hkOqG91a6W1x/PXQhE/CJ/vuHmePZxszAHFsLyM955xvAs4BnjK1/NvD1qVejSXpwkg134JYB90/yk355z2FKuj6D3rBeBnwqyaPp/pM8oaouGNn+cDb28WjNG+haOA4BLuy/7wH8I10/tmZU1UP6jrpPAx4D/BJ4clVdCd3oLLr5xb616bM0bf+hC5igJwPn0f0iM2oZXUt2M/yc/9oL6P5fvy8bR1wfCtwMeOBgVWkS3j62/PdjyzMx2tVRtwNLcgDdyLwLgQ9U1fqRbUcBX66qrw9U3sQkuYTu8W/f7n8jultVfSfJ4cCbW3u+b5KzgfvT/cf/e1V15tj2A4H/rKp9hqhvGuZ4LNaGkZgPBN5RVeMd+Be98ZF5I+v3An46CyPyFtJS/5z3I25PBp4DPJiNT084C/iHqmryDo1mmy16A0lyN+CMqjqL7j+B66mq40f2vwvwzYaG5oeujyJ0o/H2Bb5DN5fcbYYqalKq6rcBkuxG95v9mWO73JSG+2P2xh+LtWEk5nOAd0y/nKnYcIt23G50fXCbstQ/51W1LsktgYur6i+HrkeTMfLze7ylflP7D/rz26A3nFOBfZj7Ie9z+Tzd7c3x54QuVt+mG3F5DvBl4EV968fT6CaTbdWHgHcmeQHXva3zKtodeQrM/VisVo20XhbwiiRXjWzegW4Q0tenXdcULdnPOd1sCU+ju4WrNi2qn98GveGE6/8A2JwVW95lUXk5G0ccv4Su0/rngYvpJhNu1Z/QPRHlBLrpRQCuoevr0dy0C1uaO29UY/PobWi9DHAA3fyYG6wFvgq8dtpFTdGS+pyP2ZVuCqX/xdwjrZ81SFUDSXIWcNuqailvLKqf3/bRG0iS1Wx9R83Hjg3WaEqSGwE/ryXwoew7pt+6X/z+hg7rremn3JiXFp8R2l//s6uq1eljNmupfM5HJfn8ZjZXVd1nasXMgCTPBPaqqmZG1S+2n98GPUmSpEYtG7oASZIkTYZBT5IkqVEGvRnTz5235HjdS4vXvbR43UuL1z1bDHqzZyY/KFPgdS8tXvfS4nUvLV73DDHoSZIkNcpRt3NYsWxl7bxst0Hee+36q1mxbOUg733bA68Y5H0BLrrkWm681zBPg7rwmp0GeV+AK3++ll1vOMwUS5f/YNct7zQha6+5khXLB3r/ddcM877A2vW/ZMWynYd58wH/r19bV7Miw/y/RjLM+zLw3/eABr3u4f66B73uy9ZddHFV3XiubS1NYLhgdl62G4ft9gdDlzF1n/zMF4YuYRCvuuS2Q5cwiNV/fJehSxhELvrZ0CUMY20rT0/cSit23PI+akaWLc0blZ8+/y0/3NS2pfknIkmStAQY9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJatSiDHpJjkhSSfYeuhZJkqRZtSiCXpLVSd4ydB2SJEmLyaIIepIkSdp6Mx/0kpwAHA48o79dW8B+/eaDk5yW5Kokpye589ixv5PkpH77j5P8Y5Ldp3sFkiRJw5j5oAc8GzgVeCdw0/7rvH7bK4BjgDsDlwDvTRKAJAcBnwU+BhwMPAI4BHjHFGuXJEkazPKhC9iSqro0yVrgqqq6ECDJb/ebj62qz/fr/gY4GdgX+BHwAuBfqup1G86V5E+AryW5SVX9dJrXIUmSNG0zH/S24Jsjr8/vv9+ELujdBbhNkseM7JP++62B6wS9JEcBRwGszK4TKVaSJGmaFnvQWzfyuvrvy0a+vw14/RzH/Xh8RVUdDxwPsMfyvet6R0iSJC0yiyXorQV22MpjvgocWFXfm0A9kiRJM28xDMYAWAPcLcl+/STJ86n7Vf0xxyW5U5LbJHlIkrdOtFJJkqQZsViC3mvpWvXOBC4Cbr6lA6rqm8C96aZiOQn4Bt0o3Z9MrEpJkqQZsihu3VbVd4HDxlafMLbPGjYOttiw7nTgAZOsTZIkaVYtlhY9SZIkbSWDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNWr50AXMpIK69tqhq5i6g0577NAlDOIFB3x26BIG8YuD9hy6hEHc8Ix1Q5cwiKxdmtfNsqXZnlE7LtEf71dcNXQFM2dp/guQJElaAgx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowYLeklWJ3nLhN9jTZLnT/I9JEmSZpUtepIkSY0y6EmSJDVq6KC3PMkbk/y8/3pNkmUASW6Y5F39+l8m+Y8kB44enOQRSb6V5FdJzkvyl0myqTdL8rgklyV56KQvTJIkaWhDB70j+xoOA54OHAX8eb/tBODuwB8AdwOuAj6dZGeAJHcBPgB8GDgIOAZ4MfDMud4oybOBNwMPqaqPTeRqJEmSZsjygd//AuBZVVXA2UluBzw3yceBhwKHV9UXAJI8HjiXLhy+DXgucFJV/VV/ru8muS3wIrpA92tJXkYXIu9TVV+bwnVJkiQNbugWvS/1IW+DU4F9gQOA9f0yAFV1KfAt4Pb9qgOAU8bOdzKwb5LdR9Y9G/gz4J6bC3lJjkpyepLT19bV23o9kiRJM2PooLctasu7XGefk/vlP97sAVXHV9Wqqlq1Iiu3pz5JkqSZMHTQu/vY4IlDgfOBs9jYdw+AvpXuIODMftVZwD3GzndP4EdVdfnIujOA+9HdEj52YcuXJEmaXUMHvZsBb0iyf5JHAS8AXl9V/wN8FHhrknslOQh4D3AZ8L7+2NcBhyd5aZLbJTkSeB7w6vE3qaqv0IW95yV5yeQvS5IkaXhDD8Z4L7ADcBrd7dW3A6/vtz0ZeAPwMWAlXX+8B1TVLwGq6qtJ/hD4a+AvgJ8ArwTmfNpGVX05yf2Azyahqv52UhclSZI0CwYLelV1xMji9aZEqaqfA0/cwjk+TDe9yqa27ze2/GVgz60oU5IkadEa+tatJEmSJsSgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjVo+dAGaHbt+cPehSxjEXx3+8KFLGMReu2XoEgZx7Z67DF3CINbtudPQJQxi3W47DF3CIK5ZuTT/fe923q+GLmEYP9r0Jlv0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGtVU0EuyX5JKsmo+y5IkSS1bPnQBC+w84KbAxUMXIkmSNLSmgl5VXQtcOHQdkiRJs2Dmb90meUCSy5Ms75dv099+PW5kn79N8h/empUkSdpo5oMecDKwEtgQ3o6guzV7xMg+RwCrp1iTJEnSzJv5oFdVVwBnAL/brzoCeAtwiyQ3TbILcFcMepIkSdcx80Gvt5qNLXiHA58CTuvX/Q5wDfDl7XmDJEclOT3J6Wvr6u05lSRJ0kxYTEHvHkkOAHana+FbTdfKdwRwalWt3Z43qKrjq2pVVa1akZXbV60kSdIMWCxB72RgJ+CFwMn96NrVbAx6q4cqTJIkaVYtiqA30k/vccDn+9VfAn4TOBSDniRJ0vUsiqDXW003799qgKq6mq6f3q/Yzv55kiRJLVo0EyZX1THAMWPrjhhbXgNkvsuSJEktW0wtepIkSdoKBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWrU8qEL0Oy44bcvHbqEQdQOew5dwiCu+o0MXcIg1t5o5dAlDOIXt95x6BIGseKyGrqEQVy579L89737OdcOXcLMsUVPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGrXogl6SFUPXIEmStBjMfNBLsjrJPyZ5bZKLgFOSVJJHje23JsnzR5YryVFJPpDkyiTnJHnc1C9AkiRpIDMf9HqPAwLcC3jCVhz3f4CPAgcD/wK8I8nNF748SZKk2bNYgt4Pqup5VXV2VZ21FcedWFXvqarvAccC1wD3nkyJkiRJs2WxBL0ztvG4b254UVXXABcBN5lrx/427+lJTl9bV2/j20mSJM2OxRL0rhxbLrpbuaN2nOO4dXMcN+c1V9XxVbWqqlatyMptq1KSJGmGLJagN+4i4KYbFpL8xuiyJEmS5hH0kuw0n3VT9jngGUlWJbkTcALg/VZJkqQR82nRO3We66bpecA5wGrgg8DbgJ8OWZAkSdKsWb6pDUn2AfYFdu5bzTb0idsd2GUKtQFQVUfMse584IFjqz80ts94Hz6qar+FrE2SJGmWbTLoAfcHngT8JvA6Nga9y4C/mGxZkiRJ2l6bDHpV9S7gXUkeWVUf2tR+kiRJmk3z6aP3sCR7bFhIcosk/znBmiRJkrQA5hP0TgZOS/KgJE8D/h14w0SrkiRJ0nbbXB89AKrqrUn+G/g8cDFwp6q6cOKVSZIkabvMZx69xwPvAJ5AN1/dJ5McPOG6JEmStJ222KIHPBK4Z1X9FHh/ko8A7wIOmWRhkiRJ2j7zuXX7MIAku1TVVVX15SR3m3hlkiRJ2i7zuXV7WJIzgbP75YNxMIYkSdLMm8+o2zfQTZ58CUBVfQO49wRrkiRJ0gKYT9Cjqs4bW3XtBGqRJEnSAprPYIzzkvwOUEl2BJ4NnDXZsiRJkrS95tOidzTwDGBf4Md0o23/dII1SZIkaQHMp0Vv/6o6cnRFknsAp0ymJEmSJC2E+bTovXme6yRJkjRDNtmil+Qw4HeAGyd57sim3YEdJl2YJEmSts/mbt2uAHbr97nByPrLgEdNsihJkiRtv00Gvao6CTgpyQlV9cMp1iRJkqQFsMU+eoY8SZKkxWleEyZLkiRp8ZnPs27vMZ91kiRJmi1OryJJktQop1eRJElqlNOrSJIkNcrpVTZl2RIcp3LOj4auYBB7nX/x0CUMYu9dVg5dwjCSoSsYxGU333foEgZx0aoauoRBPOu+nxq6hEG896cPHLqEYZy66U3zedbtCUmu9y+lqu6zHSVJkiRpwuYT9J4/8nol8EjgmsmUI0mSpIWyxaBXVWeMrTolyZcnVI8kSZIWyBaDXpIbjSwuA+4C7DGxiiRJkrQg5nPr9gyggNDdsv0B8NRJFiVJkqTtN59bt7ecRiGSJElaWPO5dbsS+FPgnnQte18EjquqqydcmyRJkrbDfG7dvhu4nI2PPXsscCLwh5MqSpIkSdtvPkHvDlV1+5Hlzyc5c1IFSZIkaWHM5/EPX01y6IaFJHcHTp9cSZIkSVoI82nRuwvwX0nO7ZdvDnwnybeAqqo7Tqw6SZIkbbP5BL0HTLwKSZIkLbj5BL2/rarHj65IcuL4OkmSJM2W+fTRO3B0Iclyutu5kiRJmmGbDHpJXpzkcuCOSS5Lcnm//BPgo1OrUJIkSdtkk0Gvql5RVTcAXlNVu1fVDfqvvarqxVOsUZIkSdtgPn30PpXk3uMrq+oLE6hHkiRJC2Q+Qe8FI69XAncDzgDuM5GKJEmStCC2GPSq6vdHl5P8FvCGSRUkSZKkhTGfUbfjfgQcsNCFSJIkaWFtsUUvyZuB6heXAYcAX51gTZIkSVoA8+mjN/pc22uA91fVKROqR5IkSQtkPkHvX4Db9K+/V1VXT7AeSZIkLZDNTZi8PMmr6frkvQt4N3Beklcn2XFaBUqSJGnbbG4wxmuAGwG3rKq7VNWdgVsDewKvnUJtkiRJ2g6bC3oPAZ5WVZdvWFFVlwF/Ajxo0oVJkiRp+2wu6FVV1Rwrr2XjKNyZkWR1krcMXYckSdKs2FzQOzPJE8ZXJnkccPbkSpIkSdJC2Nyo22cAH07yFLpHngGsAnYGHj7pwrZGkhOAw4HDkzyjX31L4OZ0fQ0PBi4F3ge8qKrWDlGnJEnSNG0y6FXVj4G7J7kPcGC/+pNV9Z9TqWzrPBu4HV1L41/063YAPgWcCDyJbiDJ24D1wPOmX6IkSdJ0zedZt58DPjeFWrZZVV2aZC1wVVVdCJDk5cD5wJ9W1XrgrCTHAG9NcmxVXTVgyZIkSRO3Lc+6XSwOAL7Uh7wNTgZWsHEC6F9LclSS05OcvtY5oSVJUgNaDnqbM9do4uOralVVrVqRlUPUJEmStKBaCnpr6frlbXAWcGiS0Wu8Z7/f96dZmCRJ0hBaCnprgLsl2S/J3sA/ADcD/iHJAUkeDLwSeIv98yRJ0lLQUtB7LV1r3ZnARcCOwAOBOwFfB94BvJ+No3IlSZKatsVRt4tFVX0XOGxs9Rrg7tOvRpIkaXgttehJkiRphEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElq1PKhC5hJCVm+BP9oav3QFQwiy3cYuoRB1C4rhy5hGNcuzc/5Nbtk6BIGsWxdDV3CIK64dmn++971J9cMXcLMsUVPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElq1EwGvSSrk7xlgc+5Kkkl2W8hzytJkjSrZjLoSZIkafsZ9CRJkho1y0FveZI3Jvl5//WaJMsAkqxI8qokP0pyVZKvJLn/6MFJHpDk7CRXJ/kicLtBrkKSJGkgsxz0jqSr7zDg6cBRwJ/3294JHA48FrgD8C7g40kOBkjyW8C/Af8OHAK8GXj11CqXJEmaAcuHLmAzLgCeVVUFnJ3kdsBzk3wU+GNgv6o6t9/3LUnuSxcI/xT4E+DcOY5/2dSvQpIkaSCz3KL3pT6kbXAqsC9wTyDAmUmu2PAFPBi4db/vAZs4fpOSHJXk9CSnr62rF+4qJEmSBjLLLXqbU8BdgXVj63+5zSesOh44HmCP5TeuLewuSZI082Y56N09SUZa5Q4FzqdrmQuwT1V9fhPHngU8co7jJUmSloxZvnV7M+ANSfZP8ijgBcDrq+q7wHuBE5I8Ksmt+smQn5/kEf2xxwH7jR1/9BAXIUmSNJRZDnrvBXYATgP+CXg78Pp+25PpRt6+Gjgb+H/AvYEfAvSDNB4BPAD4BvAc4Jgp1i5JkjS4mbx1W1VHjCw+c47t64CX9l+bOscngE+MrX7v9lcnSZK0OMxyi54kSZK2g0FPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElq1PKhC5hJy5aRXXYeugpNy45L859BLr9q6BKGkQxdwSD2WHPN0CUM4pc32WHoEgax2w5XD13CIC7db8ehS5g5tuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1KjlQxcwK5IcBRwFsHKHGwxcjSRJ0vazRa9XVcdX1aqqWrVi2c5DlyNJkrTdDHqSJEmNWlJBL8kzk5w9dB2SJEnTsKSCHrA3sP/QRUiSJE3Dkgp6VfXSqsrQdUiSJE3Dkgp6kiRJS4lBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJatTyoQuYSct3YP0Ndx+6iqlbv9uKoUsYxJW/ufPQJQwi64euYBgrLr1m6BIGsW7npfl7/Q3WDF3BMI774IOGLmEQ+5yzbugSZs7S/JcvSZK0BBj0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRk0s6CVZnaT6r0Mn9T7zrGXNSC17D1mLJEnStEy6Re+dwE2BMwBGwtb419H99iP65bOTLB89UR/Wnj+yPBok1ya5IMmnkzwuScbquCvwyMleqiRJ0myZdNC7qqourKp1I+ueRhf+Rr/eNXbcLYCnzuP8G4LkrYCHAqcCbwU+kmSHDTtV1UXAz7b1IiRJkhaj5VveZcH9oqou3MI+bwJemuQ9VXXlZva7auRcPwK+kuRLwKeBJ9AFQUmSpCVpVgdjvBlYBzx3aw+sqs8A38JbtZIkaYkbIuidmOSKsa+Dxva5GjgWeEGSG2/De5xJdzt33pIcleT0JKevvWZzjYiSJEmLwxBB7wXAIWNf35ljvxOBNXSBb2sFqK05oKqOr6pVVbVqxfJdt+EtJUmSZssQffQurKrvbWmnqlqf5Bjg35K8cSvf4/bAOdtUnSRJUiNmtY8eAFX1SeAU4OXzPSbJ/YE7AB+cVF2SJEmLwRAtensm2Wds3RVVdcUm9n8h8CW6wRnjdunPtZxumpUH9ft/FHjPAtUrSZK0KA3RovdPwAVjX8dsaueq+gpd69xOc2x+cn/8OcDHgcOAo4GHV9W1C1u2JEnS4jLVFr2qGn9ixfj21XQDKcbXPwZ4zNi6IxayNkmSpNZMukXvqH76lLtO+H02K8l/A58asgZJkqRpm2SL3pHAzv3r8yb4PvPxIGDH/rWPQpMkSUvCxIJeVf14UufeWlX1w6FrkCRJmraZnl5FkiRJ286gJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1KlU1dA0zJ8lFwA8Hevu9gYsHeu8hed1Li9e9tHjdS4vXPX23qKobz7XBoDdjkpxeVauGrmPavO6lxeteWrzupcXrni3eupUkSWqUQU+SJKlRBr3Zc/zQBQzE615aZva6k1wxgXPul+SxzHHdI9u29dxHJPmd7Spw8mb273vCvO6lZSav2z56kjQiyRVVtdsCn/MI4PlV9ZCt2TbPc78UuKKqXrvtFUpqlS16kjSHvqVsdZIPJjk7yXuTpN+2Jsmrk3wryZeT3KZff0KSR42cY0Pr4CuBeyX5epLnjL3VdbYl2SHJa5J8Jck3kzy9P9dzkryjf31Qkm8nuT1wNPCc/vh7TfZPRdJis3zoAiRpht0JOBA4HzgFuAdwcr/t0qo6KMkTgDcAm2uRO4ZNt9pdZ1uSo/pz3zXJTsApST4LvBFYneThwF8CT6+qM5Mchy16kjbBFj1J2rQvV9WPqmo98HVgv5Ft7x/5ftgCvuf9gCck+TpwGrAXcNu+hicBJwInVdUpC/iekhpli54kbdqvRl5fy3X/z6w5Xl9D/wt0kmXAim14zwB/VlWfmWPbbYErgJttw3klLUG26EnStnnMyPdT+9drgLv0rx8K7Ni/vhy4wSbOM77tM8CfJNkRIMntkuyaZA/gTcC9gb1G+gJu7tySljiDniRtmxsm+SbwbGDDAIt/Ag5P8g2627lX9uu/CVyb5BtzDMYY3/Y24Ezgq0m+DbyVriXx9cDfV9V3gacCr0xyE+DjwMMdjCFpLk6vIklbKckaYFVVLcXneUpaRGzRkyRJapQtepIkSY2yRU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRv1/O9J8CAtBjx8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x720 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "i = 1\n",
        "plot_attention(result['attention'][i], three_input_text[i], result['text'][i])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "937148d5",
      "metadata": {
        "id": "937148d5",
        "outputId": "559f610a-ad26-46b5-ffda-3ff10a54341c"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\Nhung\\AppData\\Local\\Temp/ipykernel_31496/2069917776.py:14: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
            "  ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
            "C:\\Users\\Nhung\\AppData\\Local\\Temp/ipykernel_31496/2069917776.py:15: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
            "  ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAApYAAAJxCAYAAAD4s/IwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA/tklEQVR4nO3deZhlVXm28fvpgUZAHBDHRHFWREUEBSeIJk4xRqPRL+KERMRoYjTOiXFIjGIwUaMRccI4Jw5REyVqpFGRQUAFw6CIgAoig8gMTff7/bF3yeFY3V3dvap2nVP377rqqrOHs8+7q+pUPbX2WmunqpAkSZK21LKhC5AkSdJ0MFhKkiSpCYOlJEmSmjBYSpIkqQmDpSRJkpowWEqSJKkJg6UkDSTJIUleM3Qds0ny0CSnz3HffZL8dL5rkrT4GSwlLSlJVif5ZZJVY+vPSvK7I8s7JakkKxq97rOTfHN0XVUdWFV/1+L4rVXVN6rq7i2OleSwJH/f4liSFjeDpaQlI8lOwEOBAh4/bDWSNH0MlpKWkmcCxwCHAc+aWZnkw8DtgS8kuTzJy4Gv95sv6dft1e/7nCSn9q2e/5PkDiPHqSQHJvlhkkuSvCudewKHAHv1x7qk3/8GLXlJnpvkjCQXJ/l8kttu7NjjJ5hk6yRXJblFv/zXSa5Lsn2//HdJ3tY/XpXk4CTnJDm/vzR/o37bDS5vJ9ktyXeSXJbkP5J8crwVMslfJflFkvOS7NevOwDYF3h5f+5f6Ne/IsnP+uOdnuQRc/82SlqsDJaSlpJnAh/tPx6V5FYAVfUM4BzgD6pqu6p6C/Cw/jk37dcdneQPgVcDfwTsCHwD+PjYazwO2AO4D/AU4FFVdSpwIHB0f6ybjheW5OHAm/rn3AY4G/jExo49fpyquhr4NrB3v2rv/lgPHlk+sn/8ZuBuwK7AXYDbAX87S21bAZ+lC+Q378/5iWO73Rq4SX+M/YF3JblZVR1K9/V+S3/uf5Dk7sALgT2q6sb9eZw1/rqSJo/BUtKSkOQhwB2Af6+qE4AfAU/bxMMcCLypqk6tquuAfwB2HW21BN5cVZdU1TnAEXShbS72BT5QVSdW1TXAq+haOHfajGMfCezd9w+9D/COfnlrumD69b618wDgxVV1cVVd1p/P/5vleHsCK4B3VNWaqvoMcNzYPmuAN/TbvwhcDqyvj+ZaYBWwc5KVVXVWVf1ofV8YSZPDYClpqXgW8OWqurBf/hgjl8Pn6A7A2/tL0ZcAFwOha6Wb8fORx1cC283x2Lela1kEoKouBy7azGMfCewD7AacDHyFrqVyT+CMqrqIrsV1G+CEkfM5vF8/W20/q6oaWfeTsX0u6sP2RuurqjOAvwReB/wiySdGL/tLmlwGS0lTr+83+BS6VrufJ/k58GLgvknu2+9WY08bX4YuTD2vqm468nGjqvrWHMqY7XijzqULrjM1bwvsAPxsDsce9y261sInAkdW1Sl0fUgfy/WXwS8ErgLuNXIuN6mq2cLgecDtxvp0/vYm1PMb515VH6uqmVbkAg7ahONJWqQMlpKWgifQXX7dme7y8a7APen6SD6z3+d84E4jz7kAWDe27hDgVUnuBZDkJkn+eI41nA/8Vt9fcTYfB/ZLsms/FdI/AMdW1VlzPP6vVdWVwAnAC7g+SH6L7lL+kf0+64D3Av+c5Jb9+dwuyW/02wSOpvv6vTDJir6v6QM2oaQbfG2T3D3Jw/vzvJou4K7bhONJWqQMlpKWgmcBH6yqc6rq5zMfwDuBffu+iG8C/qa/LPzSPpy9ETiqX7dnVX2WrmXtE0kuBb4PPGaONXwN+D/g50kuHN9YVV8FXgN8mq6F8M7M3t9xro4EVnJ9X8gjgRtz/Wh3gFcAZwDH9OfzVWbpF1lV19INWNofuAR4OvBfwDVzrOX9dP0pL0nyn3T9K99M12r6c+CWdH1KJU243LDLjCRJG5fkWOCQqvrg0LVIWjxssZQkbVSSvZPcur8U/iy60eaHD12XpMWlya3KJElT7+7AvwPbAmcCT66q84YtSdJi46VwSZIkNeGlcEmSJDVhsJQkSVITBktJkiQ1YbCUJElSEwZLSZIkNWGwlCRJUhMGS0mSJDVhsJQkSVITBktJkiQ1YbCUJElSEwZLSZIkNWGwlCRJUhMGS0mSJDVhsJQkSVITBktJkiQ1YbCUJElSEwZLSZIkNWGwlCRJUhMrhi5AbSX5o8142peq6qrmxUiSpCUlVTV0DWooybpNfEoBd62qM+ejHkmStHTYYjmdbl1Vv5jLjkkum+9iJEnS0mAfy+nzIWBTLmt/BLh0nmqRJElLiJfCJUmS1IQtllMoydoktxy6DkmStLQYLKdThi5AkiQtPQZLSZIkNeGo8On1lCQbHJRTVf+2UMVIkqTp5+CdKdTPZXkl3RyV61NVtf0ClSRJkpYAg+UU6oPlnOeylCRJasE+ltPJ/xYkSdKCM1hOJ0eFL1JJ1vXTQc32cUWS7yX5i6HrlCRpczh4Zzpt8O47SXYH/r6qHr1wJan3QuB1wGeBY/t1DwSeABwE/Dbw5iRVVf8yRIGSpMUtyc039TlVdfF81DLOPpZTKsnvAY8E1gDvq6ozk9wN+EfgccBXDJYLL8nngM9X1fvH1u8PPL6q/jDJgcCfV9W9BilSkrSo9WMpNiXAFXC3qjpznkr6NYPlFEryLOCDwMXAzYELgBcB7wE+A/xTVZ08XIVLV5LLgV2r6oyx9XcBvldV2ya5M3ByVW0zSJGSpEWtD5ZPovs7v9HdgS8CuyxEsPRS+HR6MfDqqnpzkqcAnwBeBuxWVT8atrQl7yK6y94Hj61/AnBh/3g74FcLV5IkacKcDXy9qi6ay85JzqS7gjnvDJbT6c7AJ/vHnwLWAi8xVC4Krwfem+ThwHH9uj3oui08t1/+PeDIAWqTJE2AqrrjJu6/y3zVMs5L4VNofB7LJJcB912IJnBtXJK9gD8H7tGvOg14R1UdM1xVkiRtOVssp9fvJ5m5nLoMeFSS80d3qKrPLHxZqqqjgaOHrmNLJbk1sKKqfjq2/reANVV1/uzPXLz6luSd+8VTq+p/h6xHkjYkSYBn0vW3vBPdIJ0zgf8APloDtB7aYjmF+hbLjamqWj7vxWhWSW4L3JKxuWSr6sRhKtp0Sb4KfLKq3ju2fn/gqVX1yGEq23RJ7gh8GrgPcG6/+rbAycCTbO2XtBgl+QxdH/2TgVPoBursDOwCfLaqnrTgNRkspYWT5H7AR+gug49PZD9RYT/JJcADq+r0sfV3A46pqk2eZ20oSb4GLAeeUVXn9OtuTzcnbFXVw4esT5LGJdmXbraXP6qqL49texTdP8sHVNXHFrQug+XSlOR3q+qrQ9ex1CT5Nt3I8DfQtYzd4A1YVWcPUdfm6KdOelBVnTS2/j7A0VW17TCVbbokVwF7VtX3xtbvSncuNxqkMElajyRfAr5VVX+3nu2vpfvn/7ELWZd9LJeQJLcD9gOeA9yBroVGC2tn4H5V9YOhC2ngWOD5/ceoFwDfXvhytsg5wGzhcWvgJwtciyTNxX2Bv9nA9v8GnrdAtfyawXLKJVkO/CGwP92UNicBh9B17NXCOxm4NTANwfKvga/1LZRf69c9HLgf8LuDVbV5/gp4R3+f9m/TtSQ/AHhbv21iJHki8DvM3of3KYMUJWk+7ACct4Ht59HdJGVBeSl8SiW5O/CndKPFrgA+BryCbtqhU4asbSnrRx3/A91/mSczNmHtQt3LtZUk9wVeDuzar/oO8I/jl5QXu35KrlV0rfgzg9+W0c0Be/XovlW1/cJWN3dJ3ko3ldVRwPn8ZleLPxmiLknt9QN1b1VVF6xn+62Acxe6777Bcgol+QbdiLBPAx+uqiP79WswWA5qbMT+6JsvTNjgnWnS3wZ1TqrqQ/NZy5ZIciGwf1V9buhaJM2v/u/JB4Ar17PLNsB+C/13xUvh02kv4F3AoVX1f0MX08o0TNFDd4lyaiRZBexL13e0gP8DPl5V1wxa2CZazGFxE11JN+G+1FySXavqu0PXoV/7Ot2d9ja2z4KyxXIK9VPa/CnwNOAs4N+Aj9MNQpi4FstpmqJnmiTZGTgc2J7usj7Avenuc/7oqjp1qNo2R3/Z6Bl0v6hfU1UXJnkw3aWkHw9b3dwk+TPg/sDzquq6oevRdOlbyL4DvA/4WFX9aiNP0RJksJxiSbYG/phuFPhD6Fr6Xgm8r6p+OWRtm2KapuiBXweYF3DDVr53T9qdapJ8ha6F7BlVdWm/bnu6fwJWVdWjhqxvUyS5P/C/wI+BewH3qKozk7wOuFtVPW3I+uYqyUrg88BudAPExvvwOh+nNluSu9L9PXkGcDPgs8D7q+qIQQvTomKwnEL9xM4/Gb2VU5K7cP1gnh2Ar1XVYwYqcZMkuYIpmaKnbwE7nG5gxcxtHfeiu8T/qP52jxMhyZXAHuPdLZLcm26C9Emax/II4OtV9dp+IM99+2C5F/CJqrrDwCXOSZL3A0/h+p+x8X/C/nyIuuYiyTuAV1XVFf3j9aqqv1igsjSLJMuAx9BNX/cHwE/p+vp9aPwWr5o/SV4yl/2q6p/mu5ZRBssplGQtcJuq+sUs25YDjwOeU1V/uODFbYYkxwAvr6oF7yvSWpKj6S4bH1hV6/p1y+imgNqlqh40ZH2bIsnFwB9U1VFj6x8CfK6qdhimsk2X5FJg1z5MjgbLnYDTqmrrYSucm37S+idW1VeGrmVT9eH+iVV1Sf94fbwT0iLRXxV7PvAmYCvgOuAzwF9V1c+GrG0pSLKhLjpFN7XdKgfvqIXxfoi/VlVrgc/1H4tWktG5t14NvCXJNEzRsyvw7JlQCVBV65L8E13fpUnyBeC9SZ4LHNOv24vuFmOfH6yqzXMV3aW9cfcAfuMftEXsAmAi/6BX1e/M9liLT5IH0F0SfypwKfBmuhbL29B1WfpPYI+h6lsqquqOs61PcifgjXRd4RZ8zmqDpRarC/nN6Xi+PMu6YrLuIPQr4I7A6WPr7whcsuDVbJkX0d1L+xt08z1C14/388BfDlTT5voc8Nokf9wvV99aeRDdtF2T4rXAG5I8u6ouH7oYTZf+0ut+wN3o7uqyL3D4yD/K5yTZn27QqBZYkh2A1wAH0s1lu2dVHb/gdXgpfPr0I/cOBjb4h6Wq3rAwFW26JHvPdd+ZeTonQZK30f0X+XLgW/3qB9MFmE9W1Zz6zCwmfYf+e/SLp1bVGUPWszn6QUdfBO4DbAv8HLgV3S/nx1bVFQOWN2dJTgZ2ogv45/Cbrfv3GaCszZLkbsCTgdvTXWb9tap6ziBFLXFJfgi8H/jg+gYbJtkK+JMpmsJr0UtyI+AldH9XzgJeWVVfGqweg+X06YPl6XT9XdanJumPzLTof+n+I91/lDNXDNYA7wZeUVXXDlWbfn1npN3ogtmJVfXVgUvaJEleu6HtVfX6haplSyT5fbqW4u/QTZ/0bbppoFYB36iqxw9YnrQo9P3z9wdeT/d35DV0N0UZNNgZLKdQHyxvPdvgnUmVZBu6/omzTZD+mSFq2hL9+cxMbPujqlrfnRMWLUfuar4kOQH4VFW9aWYwFd1UYx8Gjl7oUa66of5mFbO1JE/8AMtJkuQU4A7AO4B/Yez2szMWehyCwXIKbWhU+CRK8rt0E7zPNsrYCdIHMsvI3ZV0l8SXA9+ZtJG7SR4IPILZ/3kxJC+gfnT7ffqR+RcDD6uq7/dTWf13Vd1+4BKXpD5Qfhx4KF3/9pl+7gD4u3hhbeAWwb/ehQH+Rjp4Zzqtd1T4hHo7XUfxV1fVuUMXsyX66TlexPoDzMR0T5ht5G5/fu+nG9AzMZK8FHgLcAa/OQn/xPz33Xe1+GvgT+halFaObp+gP/yXATNTPJ0H3AX4Pt3frNlG72thvI2ui9XOdN0THk3XF/kNwIuHK2vJWpSzJxgsp9Pr2cjAnQmzE/D4SQ+VvX8Fnkg3BcS3mKDQMhdVdXWSf6CboPuQoevZBC8C/qKq3jl0IVvo7+imgHkT8M/Ay+jeP/+Prv/VpDiW7m5hp9D9U/nWJPele+9MzE0EptDewO9X1WlJCrigqo5Kcg3dz97EzZ86yRbrwFWD5XR6K3AjutvtAZDknnR/ZLYDPlNVnxiots1xFHB34EdDF9LAE4A/nrRBIZvoFnQ/Z5NkZlT4pHsK3eT7hyc5mG6i+h8lORX4Pbo5RifBS7j+Z+h1wI2BJ9HdpnLiZk6YIjeimwoO4GK6qy4/oPsHYGKutkyLJAfQ3e3omn75XsDpVXVdv7wt3aDQv13IugyW0+nddPMlvhAgyS3oLk2uo7us9NEky6rqY8OVuEkOAQ7u+/fMNkH6iYNUtXmuBH4ydBEtzHI7sdBNkLwvkxfSPk53We9fhy5kC92K7o88dFctbto/PpxuSqtFL8kKur66xwL0A9ueP2hRmnEa3ffmLOC7wIFJfgK8gAmdmH/CvZtuMvqZ8RRH0w1yPbNf3o6ua4zBUltsL+B5I8vPAK4F7llVv0pyEF3onJRg+an+86GzbJu0CdLfArwkyYFDTwnRwPh9p9fR3fnlg3SXYhe1sWD8E+D1/b3cT+I3/3mZlFHI5wC37T+fATwKOIHud8JVA9Y1Z1V1XZLP0AWYi4auRzfwdrrbBELXr/Jw4GnANcAzhypqCRsfT7EoxlcYLKfTbbjhZePfAT5dVb/qlz9EdzuuSTHrbasmRZLx2xs+DHh0P1XEeIBZ9PPzJfkQ8LL13U5sgowH48uBB/UfowqYlGD5WbqBYcfQhYCP97fcvB3d/KmT4nt0A3bOGrgOcYP3/Edn1lXVif3dqe4BnFNVF67v+VpaDJbT6Uq6u4fMeADwyZHlq4FtFrSiLVBVZ/eXxx7Ab86dVnRz2y1m460unx2kigaSfJBuIMg2/fyC67XY746ygfvsbtdvn7gBcFX1qpHHn+ovUz4Y+EFV/ddwlW2y19EN2HktXYvrDe58tNDz8rWQ5FZ0l4x3pvu9dQrwr+u7g81iMdf3fJJF/57XwjBYTqfv0d3P9aVJ9gF2BL42sv3OdFOqTIQk9wC+QNdyGbr7Uq+ga+27hkUeLKtqv5nHSQ6oqtku6ZNkEkZR70j3Pbg53XQwD6O7BH5yv30XuimUJm6i5CR/STcw5Hb98rl0LZVvm5RuC0neCPykqg4BqKpjgWOTHJjk76pqUkaG/3f/+TPccOaEmXkTJ6n7C30Xi8OB87l+VPu+wIuTPKqqFvNI96l9z0+J308yczVyGfCoJDP/rNx0iIKcIH0K9ffZ/hLd6L0dgY9V1f4j2/8VuNFo4FnMkhwOXEJ366qf03VOvgldx+W/qaqJmeIiyS+BP62qT4+tPwR4TFXdYZjK5i7JpXTfg6cC9wP2m7mXdj8K8f3AyVX1xsGK3ERJ3gIcQHe5eOaP/F7AS4H3VtXLh6ptUyQ5h27WgWPH1u9BdyebRf/zBZDkWXT9XteObVoG3H7S7kOd5Gi6IHZgVa3r1y2jG5i4S1WNd79YVKbxPT8NxiZIX58FnyDdYDml+umFHkkXxP5j5pdZv+0A4Liq+u5A5W2SJBcBe/d33vgV8ICqOr0P0P8ySZOKJ3kEXSvMH1XV//brDqUbkbxPVZ25oecvBklOoxsUcgzwiKo6ZWz7vYD/rapbz/b8xai/u8sBVfWpsfVPBt5TVbPd9WnRSXI1sPP4z1GSOwGnVNXWsz9zcVnf3cOS7AD8YoImegcgyVXArlV1+tj6e9DdpepGw1Q2N9P4ntf8WbbxXTRJkjwgyfKqOrWq3l5VnxwNlQBVdehMqExy/yQrZz3Y4hGun5PzAvpLlcBP6Tr4T4w+TO4PfCrJA5O8l+4X9kSESoCqukdVnU03lcVtZ9nlNkxQH94RJ61n3ST9njyH7nZ74x5G936ZFDe4VeCI7VjP/ZAXuV8x+yDEO9JdjVnUpvg9P7Fm/tZvwv4L9rfePpbT52i66SAumOP+R3DDea8Wo+8D96Wr8TjgFX2LxnPpplSZKP2gipvR9Uk6j6419qxhq9osnwY+mORldC0ZAHvSzZf4mcGq2jz/Rjew4kVj65/PIu/DO+Y9wD/3t3ac6Vf9CLrpnxb9PJZJ3tE/LOBNSa4c2bycbgDfdxe6rgY+Abw/ycvp7rgF3aCqg+jmUJ0U0/Sen3SL9m+9wXL6hN/8hbwhW218l8G9ketHuf8NXcf+I+j6kD5lqKLmauSP5bhf0PW7eknSTT9WVX+xUHU18Hy6uzwdxvX3pL6Orr/VSweqaXOtAp6WZOZyH8AD6VpnPjr6PVzM36Oqemt/Q4R3cP17+1rg7VX1luEqm7N7958D3JOu9hnXAicCBy90UQ28nO6cPsD1f3fX0PUTf+VQRW2GaXrPr1d/p6q7VtVizkiL9m+9fSynTJLVbPr9p59WVefNQznzJsnNgV9OwmjdJEfMcdeqqofPazHzoO+8f+d+8UcznfonybR9j/rvyc794qmTNnVSP8XNi6rq0qFraSnJNtzwvTLXULCoTMN7fkOSvBDYoapeP3Qt67OY/9YbLCVJktTEJHVKlyRJ0iJmsJQkSVITBsslpJ+/cuJNy3mA57JYTcu5TMt5gOeyWE3LuUzLecDw52KwXFqm5Y0zLecBnstiNS3nMi3nAZ7LYjUt5zIt5wEDn4vBUpIkSU04KnwR2CqrautfT9M4f9ZwDStZNb8vst3835ns2jVXsNXK+f163e1OF83r8WdccNFadtxhfu9O98OT5/9nC+BarmGr+f756uf7nG/X1tVslfm9+2Gtm8ttfrfMgrznF4jnsmmyfGHaja5ddzVbLZu/90qtW5iMsqauZuU8v+c3fXagzbOmrmFl5vfn67L65YVVteNs2xbz5J9LxtZsywOXP3LoMppYd797b3ynCfCVfz9s6BKaecyd9hy6hHaWT9Qtojdo3VWTeGfCWaxbO3QF7Sybnp+v5dstzD+U823dNdcMXUI7a6fnvfKVNZ84e33bvBQuSZKkJgyWkiRJasJgKUmSpCYMlpIkSWrCYClJkqQmDJaSJElqwmApSZKkJgyWkiRJasJgKUmSpCYMlpIkSWrCYClJkqQmDJaSJElqwmApSZKkJgyWkiRJasJgKUmSpCYMlpIkSWrCYClJkqQmDJaSJElqYskHyyTLkrwnyUVJKslZSf5r6LokSZImzYqhC1gEHgvsB+wDnAlcBWRLDpjkMOAWVfW4LS1OkiRpUhgs4S7AeVX1rbnsnGSrqrp2nmuSJEmaOEs6WPYti8/qHxdwNrCakdbGJKuBU4Er+n3PAvZI8jzgr4DbA5cDJwC/D/zN2DEBfqeqVs//GUmSJA1nSQdL4EV0YfI5wB7AWuAfZ9nv6cChwEOBJNkdeBddgPwmcFPg4f2+BwP3BG4OPKNfd/H8lC9JkrR4LOlgWVW/SnIZsLaqfg6QzNq98sdV9VczC0n+iK4F8/NVdRldOP1ev/nyJFcB18wcczZJDgAOANiabVqcjiRJ0qCW/KjwOTphbPkrdGHyx0k+muRZSW68KQesqkOraveq2n0lq5oVKkmSNBSD5dxcMbrQt1LuBjwFOAd4FXBaktsOUJskSdKiYLDcTFV1XVV9rapeBdwH2BaYmV7oWmD5YMVJkiQNYEn3sdxcSR4H3Bn4Ot3AnN8Bbkw3ehy6keOPSXJ34CLgV1W1ZoBSJUmSFowtlpvnEuAJwFeB04CXAn9aVd/ot7+XLmQeD1wAPHjhS5QkSVpYS77FsqoOppsiaGb52WPb95nlOd+ka6Vc3zEvAB7ZrEhJkqQJYIulJEmSmjBYSpIkqQmDpSRJkpowWEqSJKkJg6UkSZKaMFhKkiSpCYOlJEmSmjBYSpIkqQmDpSRJkpowWEqSJKkJg6UkSZKaMFhKkiSpCYOlJEmSmjBYSpIkqQmDpSRJkpowWEqSJKkJg6UkSZKaMFhKkiSpiRVDF6DpsvKH5w5dQhOPOOXxQ5fQzKrfnp63+aX32XHoEprZ/ms/GLqEJtZecsnQJTSzbKuVQ5fQTLbdZugSmsjatUOX0E4ydAXtrFn/JlssJUmS1ITBUpIkSU0YLCVJktSEwVKSJElNGCwlSZLUhMFSkiRJTRgsJUmS1ITBUpIkSU0YLCVJktSEwVKSJElNGCwlSZLUhMFSkiRJTRgsJUmS1ITBUpIkSU0YLCVJktSEwVKSJElNGCwlSZLUhMFSkiRJTRgsJUmS1ITBUpIkSU0YLIEk/5XksKHrkCRJmmQGy3mQ5HVJvj90HZIkSQvJYClJkqQmllywTLJNksOSXJ7k/CSvHtt+syQfSvLLJFcl+WqSe41sf3b/3Eck+X6SK5IckeSOM9uB1wL3SlL9x7MX8hwlSZKGsOSCJXAw8HvAk4BHAPcDHjay/TDggcAfAg8ArgQOT3KjkX1WAa8CngPsBdwUOKTf9kngrcDpwG36j0/Oy5lIkiQtIiuGLmAhJdkO2B94TlX9T79uP+Cn/eO7Ao8H9q6qr/frngGcA+wLvK8/1ArgBVV1er/PwcAHkqSqrkpyOXBdVf18A7UcABwAsDXbND9XSZKkhbbUWizvDGwFHD2zoqouB07uF+8JrBvb/qt++84jx7lmJlT2zu2Pe7O5FlJVh1bV7lW1+0pWbep5SJIkLTpLLVhuiRp5fN16tvn1lCRJS9ZSC0I/AtYAe86sSLItsEu/eCrd12Svke3bA/cGTtmE17kWWL6lxUqSJE2SJRUs+8ve7wcOSvJ7/WjvD9CHwKr6IfA54D1JHprk3sBHgEuBj23CS50F3CHJbklukcRr3ZIkaeotqWDZeylwBPDZ/vP3ga+PbN8POA74fP95G+DRVXXVJrzGp4EvAv8LXAD8yZaXLUmStLgtqVHhAFV1BfDM/mO27b8EnrWB5x9GNyXR6LrVQEaWrwGevMXFSpIkTZCl2GIpSZKkeWCwlCRJUhMGS0mSJDVhsJQkSVITBktJkiQ1YbCUJElSEwZLSZIkNWGwlCRJUhMGS0mSJDVhsJQkSVITBktJkiQ1YbCUJElSEwZLSZIkNWGwlCRJUhMGS0mSJDVhsJQkSVITBktJkiQ1YbCUJElSEyuGLkC9Wjd0BW1cc83QFTRx9vm3G7qEZn7rrtPzNr/yltPzv/D2K6bj+5Lly4cuoZlaOyW/h4G1F148dAkaU2vXDl3Cgpie39KSJEkalMFSkiRJTRgsJUmS1ITBUpIkSU0YLCVJktSEwVKSJElNGCwlSZLUhMFSkiRJTRgsJUmS1ITBUpIkSU0YLCVJktSEwVKSJElNGCwlSZLUhMFSkiRJTRgsJUmS1ITBUpIkSU0YLCVJktSEwVKSJElNGCwlSZLUhMFSkiRJTUx1sEyyOsk7h65DkiRpKZjqYClJkqSFY7CUJElSE0sqWCZ5RJJLkhyY5LAk/5XkRUl+luSXST6YZJuR/VcleVuS85NcneSYJA8Z2X5MkleOLH8kSSW5db+8TZJrRp8jSZI0rZZMsEzyZOCzwAFVdUi/+qHALsDvAk8Fngi8aORpb+nXPwe4H3AycHiS2/TbVwP7jOy/N3DhyLoHAdcBx81SzwFJjk9y/Bqu2cKzkyRJGt6SCJZJDgDeDzy5qv59ZNOlwIFVdWpVfRn4D+AR/XO2BZ4PvKKq/ruqTgUOBM4HXtA/fzXwkCQrktwFuAnwHuB3+u37AEdX1bXjNVXVoVW1e1XtvpJVbU9YkiRpAEshWD4BeBfw6D48jjqlqtaOLJ8L3LJ/fGdgJXDUzMZ+36OBnftV3wRWAXvQhchvAl/l+hbLfejCpyRJ0tRbCsHye8B5wP5JMrZtzdhyMbevSQFU1eXACXQtlPsARwDHALfvWzD3wGApSZKWiKUQLH9MF/oeCRw6S7hcnx8B1wIPnlmRZDmwF3DKyH6r6YLl3sDqqroaOBb4a9bTv1KSJGkaLYVgSVWdSRf+Hg28Zy7hsqquAN4NHJTksUnu2S/fCvjXkV1X0wXX7YETR9Y9nfX0r5QkSZpGSyJYAlTVj+gC4GPoBtjMpeXyFcAngQ8C3wXuQ9dX87yRfb7Zf/7GSH/N1cAKvAwuSZKWkBVDFzCfqmqfseUfAb+9gf1fB7xuZPka4C/7j/U953K6QT6j61Yzt+AqSZI0NZZMi6UkSZLml8FSkiRJTRgsJUmS1ITBUpIkSU0YLCVJktSEwVKSJElNGCwlSZLUhMFSkiRJTRgsJUmS1ITBUpIkSU0YLCVJktSEwVKSJElNGCwlSZLUhMFSkiRJTRgsJUmS1ITBUpIkSU0YLCVJktTEiqELEJCQFSuHrqKNdTV0BU3c7S/OHrqEZrL9jYcuoZmLdrnd0CU0c+UDdhq6hCa2+eH2Q5fQTG2zaugSmln2y8uGLqGN5cuHrqCdK68auoJ2zlv/JlssJUmS1ITBUpIkSU0YLCVJktSEwVKSJElNGCwlSZLUhMFSkiRJTRgsJUmS1ITBUpIkSU0YLCVJktSEwVKSJElNGCwlSZLUhMFSkiRJTRgsJUmS1ITBUpIkSU0YLCVJktSEwVKSJElNGCwlSZLUhMFSkiRJTRgsJUmS1ITBUpIkSU0YLMckWZ3knUPXIUmSNGkMlpIkSWrCYLkAkqwcugZJkqT5ZrCc3bIk/5DkwiS/SHJwkmUASbZKclCSnya5Msm3kzxq5olJ9klSSR6b5Lgk1wKPWu8rSZIkTQmD5ez2Ba4DHgS8EPhL4Kn9tg8CewNPA3YBPgR8Icl9x45xEPA3wD2AY+e/ZEmSpGGtGLqAReqUqvrb/vEPkjwXeESS44A/AXaqqnP67e9M8rvA84A/GznG66rqy+t7gSQHAAcAbM02zU9AkiRpoRksZ3fS2PK5wC2B3YAApyQZ3b4K+NrYc47f0AtU1aHAoQDbL9uhtqRYSZKkxcBgObs1Y8tF121gWf94j1n2uWps+Yr5KU2SJGlxMlhumu/QtVjeuqqOGLoYSZKkxcRguQmq6gdJPgocluSvgBOBmwP7AGdW1WeGrE+SJGlIBstNtx/w18BbgN8CLgaOA2zBlCRJS5rBckxV7TPLumePPF4DvK7/mO35q+kul0uSJC0pzmMpSZKkJgyWkiRJasJgKUmSpCYMlpIkSWrCYClJkqQmDJaSJElqwmApSZKkJgyWkiRJasJgKUmSpCYMlpIkSWrCYClJkqQmDJaSJElqwmApSZKkJgyWkiRJasJgKUmSpCYMlpIkSWrCYClJkqQmDJaSJElqYsXQBQiSkK1XDV1GE7V27dAlNJFV0/H9mDbb/3g6fr4Arrr5dPz63faqa4YuoZnaeuXQJTSz9ubbD11CE8suu3LoEprJ8uVDl7AgbLGUJElSEwZLSZIkNWGwlCRJUhMGS0mSJDVhsJQkSVITBktJkiQ1YbCUJElSEwZLSZIkNWGwlCRJUhMGS0mSJDVhsJQkSVITBktJkiQ1YbCUJElSEwZLSZIkNWGwlCRJUhMGS0mSJDVhsJQkSVITBktJkiQ1YbCUJElSEwZLSZIkNbHkg2WSs5K8dBP23ylJJdl9PuuSJEmaNCuGLmAR2AO4ouUBk+wDHAHsWFUXtjy2JEnSYrWkg2WSrarqgqHrkCRJmgYbvRSeZNVc1k2CJKuTvDvJwUkuAI4avxSe5G5JjkxydZLTkzw2yeVJnj12uDsk+UqSK5OckuT3+ufvRNdaCXBBf9n8sAU4PUmSpEHNpY/l0XNcNymeDgR4KPDM0Q1JlgGfBa4D9gSeDbwWmC1IvxF4B3Bf4NvAJ5JsB/wEeFK/z72A2wAvan0SkiRJi816L4UnuTVwO+BGSe5HF8YAtge2WYDa5suPq+qvZhaSjG77PeDuwCOr6mf99hcDR81ynH+uqi/0+7yaLqTuWlXfTHJxv88v1tfHMskBwAEAW2fbLTsjSZKkRWBDfSwfRddi91vAW7k+WF4KvHp+y5pXJ2xg2z2Ac2dCZe/bwLpZ9j1p5PG5/edbzrWIqjoUOBTgJstvUXN9niRJ0mK13mBZVR8CPpTkSVX16QWsab61GgG+ZuZBVVXf8rnkp2+SJElL11yC0BOS3GRmIckdkvzvPNY0pNOA2ya57ci63dn0wHht/3l5k6okSZImwFwC0zeBY/vR0c8FvgK8bV6rGs5XgNPpWmrvm2RP4J/oBvNsyuXqs/v9fz/Jjv2gHkmSpKm20WBZVe8B/hT4HPAG4GEzg1amTVWtA55INwr8OOBDdKO/C7h6E47zM7rR5G8Ezgfe2bxYSZKkRWajE6QneQbwGrpRz/cBvphkv6r63nwX11pV7TPLup3Gln8APGxmOcl9gZXAGf32s7h+INPo8zK2/HfA32151ZIkSZNhLnfeeRLwkKr6BfDxJJ+la8nbdT4LG0qSJ9IN8PkhsBPdpfDvAScOWJYkSdKit9FgWVVPAEiyTVVdWVXHJXnAvFc2nBsDBwG/DfwSWA28uKqcEkiSJGkD5nJLx72SnEI3Ynrm0vDb5rmuwVTVv1XV3arqRlV126p6WlWdP3RdkiRJi91cRoW/jW6y9IsA+r6VD9vQEyRJkrT0zGl+xqr6ydiqtfNQiyRJkibYXAbv/CTJg4BKshJ4EXDq/JYlSZKkSTOXFssDgRcAtwN+Rjca/M/msSZJkiRNoLm0WN69qvYdXZHkwcBR81OSJEmSJtFcWiz/ZY7rJEmStIStt8UyyV7Ag4Adk7xkZNP2wPL5LkySJEmTZUOXwrcCtuv3ufHI+kuBJ89nUZIkSZo86w2WVXUkcGSSw6rq7AWsSZIkSRNoo30sDZWSJEmaizlNkC5JkiRtzFzuFf7guayTJEnS0uZ0Q5IkSWrC6YYkSZLUhNMNSZIkqQmnG1oEqtZRV101dBlN1LoauoQmct11Q5fQzqWXDl1BMzf5zvSMN7zuVjcZuoQm6rLLhi6hmbW/vcPQJTTz8722HbqEJm5+2jZDl9DM1r+4eugS2vnp+jfN5V7hhyX5jbRQVQ/fgpIkSZI0ZeYSLF868nhr4EnAFDXnSJIkqYWNBsuqOmFs1VFJjpuneiRJkjShNhosk9x8ZHEZcH9gOjoHSZIkqZm5XAo/ASggdJfAfwzsP59FSZIkafLM5VL4HReiEEmSJE22uVwK3xr4M+AhdC2X3wAOqaopGjcvSZKkLTWXS+H/BlzG9bdxfBrwYeCP56soSZIkTZ65BMtdqmrnkeUjkpwyXwVJkiRpMs3lNhYnJtlzZiHJA4Hj568kSZIkTaK5tFjeH/hWknP65dsDpyc5Gaiqus+8VSdJkqSJMZdg+eh5r0KSJEkTby7B8u+r6hmjK5J8eHydJEmSlra59LG81+hCkhV0l8clSZKkX1tvsEzyqiSXAfdJcmmSy/rl84HPLViFkiRJmgjrDZZV9aaqujHwj1W1fVXduP/YoapetYA1SpIkaQLMpY/ll5I8bHxlVX19HuqRJEnShJpLsHzZyOOtgQcAJwAPn5eKJEmSNJE2Giyr6g9Gl5P8NvC2+SpIkiRJk2kuo8LH/RS4Z+tCJEmSNNk22mKZ5F+A6heXAbsCJ85jTfMuyWHALarqcUPXIkmSNC3m0sdy9L7g1wEfr6qj5qkeSZIkTai5BMtPAnfpH59RVVfPYz2SJEmaUBuaIH1FkrfQ9an8EPBvwE+SvCXJyoUqcL4lWZXkbUnOT3J1kmOSPGRk+z5JKskjkhyb5MokxyfZbew4z0lyTr/9C0n+LEn95itKkiRNpw0N3vlH4ObAHavq/lW1G3Bn4KbAwQtQ20J5C/BU4DnA/YCTgcOT3GZsvzcBrwR2Ay4CPpokAEn2At4HvIuuD+rngdcvRPGSJEmLxYaC5eOA51bVZTMrqupS4PnAY+e7sIWQZFu683lFVf13VZ0KHEh328oXjO3+mqo6oqpOA94A3AO4Xb/tL4AvV9VBVfWDqnov8NmNvPYBfcvn8WvqmpanJUmSNIgNBcuqqt+4lFtVa7l+lPikuzOwEvj1YKT+/I4Gdh7b96SRx+f2n2/Zf74HcNzY/sdu6IWr6tCq2r2qdl+ZVZtatyRJ0qKzoWB5SpJnjq9M8nTgtPkradEYD89rZtm2OfOASpIkTaUNjQp/AfCZJM+hu4UjwO7AjYAnzndhC+RHwLXAg/vHJFkO7AV8bBOOcxqwx9i6B7QoUJIkaVKsN1hW1c+AByZ5OHCvfvUXq+p/F6SyBVBVVyR5N3BQkguBHwMvBm4F/OsmHOodwDeTvAz4T+BhTE/4liRJmpO53Cv8a8DXFqCWobyi//xBuhHv3wEeXVXnzfUAVXV0kufSjQR/A/BV4CDg79uWKkmStHjNZYL0qVNVzx55fA3wl/3HbPuuBjK27qxZ1n0A+MDMcpJ/Bs5oUrAkSdIEWJLBcj70l8G/AlwO/C7dtEWvHrQoSZKkBWSwbGd34KXATej6ar4KePugFUmSJC0gg2UjVfXUoWuQJEkakvMwSpIkqQmDpSRJkpowWEqSJKkJg6UkSZKaMFhKkiSpCYOlJEmSmjBYSpIkqQmDpSRJkpowWEqSJKkJg6UkSZKaMFhKkiSpCYOlJEmSmjBYSpIkqQmDpSRJkpowWEqSJKmJFUMXIIBApiPjL9tqOs6DlSuHrqCZuvbaoUtoZ+X0/Mpads11Q5fQRLbddugSmll7o+n5+dr23HVDl9DE2lVT8jcFYN10fE82Zoq+Y5IkSRqSwVKSJElNGCwlSZLUhMFSkiRJTRgsJUmS1ITBUpIkSU0YLCVJktSEwVKSJElNGCwlSZLUhMFSkiRJTRgsJUmS1ITBUpIkSU0YLCVJktSEwVKSJElNGCwlSZLUhMFSkiRJTRgsJUmS1ITBUpIkSU0YLCVJktSEwVKSJElNLIpgmWSfJJXkFkPXIkmSpM2zKIKlJEmSJp/BcgEkWZZk+dB1SJIkzad5C5ZJHp3ksiQr+uW79Je7DxnZ5++TfHXkafdNcmySK5Mcn2S3sWM+KMmR/fafJXl3ku1Htq9O8q9J/iHJhUl+keTgJMtG9tkqyUFJftof59tJHtVvW5bkJ0n+fOx179bXvlu/fJMkh/bHv6yvafeR/Z+d5PIkj03yfeBa4J5NvrCSJEmL1Hy2WH4T2BqYCVz7ABf2nxlZt3pk+U3AK4HdgIuAjyYJQJJ7A18GPg/cF/gjYFfgA2Ovuy9wHfAg4IXAXwJPHdn+QWBv4GnALsCHgC8kuW9VrQM+3h9j/JinVtWJfT3/DdwOeBxwP+DrwNeS3GbkOVsDrwGeB+wMnD16wCQH9OH5+DV1NZIkSZNu3oJlVV0OnAD8Tr9qH+CdwB2S3CbJNsAe3DBYvqaqjqiq04A3APegC3AALwM+WVVvraofVtWxwPOBJyW55cgxTqmqv62qH1TVvwNHAI8ASHJn4E+Ap1TV16vqzKp6J/BFugAI8BHggf2+M57Wr6c/n12BJ1fVcVV1RlW9BjgTeMbIc5YDL6yqo/paLhv7+hxaVbtX1e4rs/XGv6CSJEmL3Hz3sVzN9S2UewNfAo7t1z2IrmXxuJH9Txp5fG7/eSY03h94en+J+fIklwNH9dtGQ+DoMWaOM3OM3YAAp4wd5/dnjlFVJwEn07daJnlgv+2jI3VsA1wwdoxdxuq4DvjubF8USZKkabRino+/GnhhknsC29O1YK6ma/X7BXB0VV3bX+0GWDPy3Oo/Lxv5/D7gn2d5nZ+NPF4ztq3GjlF0LaXj+1018vgjwP50rab7At+sqplL2cuA84GHzlLHpSOPr6mqtbPsI0mSNJXmO1h+E1gFvJwunK1Nshp4L104O3wTjnUicK+qOmML6vkOXYvlravqiA3s9zHgTUn2pOuf+ZqxOm4FrKuqM7egFkmSpKkyr5fCR/pZPp2uryPAMcBvAXtyw/6VG3MQ8IAkhyS5Xz/K/HFJ3rMJ9fyA7pL2YUmenOROSXZP8tIkfzSy30+BI4FDgJsA/zFymK/SXYL/XJLHJLljkr2SvD7JbK2YkiRJS8JCzGO5mq5ldDVAVV1N18/yGm7Yv3KD+r6PDwN2ogt936MbRX7+JtazH93I8LcApwH/1R/37LH9PkI3+vyLVfXLkToKeCzwNbqW19OBfwfuzvX9QiVJkpacdDlJQ9p+2Q6158pHD11GE1k+JXPur1w5dAXN1LXXDl1CM8vudPuhS2hm3barhi6hieXnXTx0Cc1cfY/bbHynCXHlLafjd9iKa6Yno2x79uVDl9DMV45//QlVtfts26YkBUiSJGloBktJkiQ1YbCUJElSEwZLSZIkNWGwlCRJUhMGS0mSJDVhsJQkSVITBktJkiQ1YbCUJElSEwZLSZIkNWGwlCRJUhMGS0mSJDVhsJQkSVITBktJkiQ1YbCUJElSEwZLSZIkNWGwlCRJUhMGS0mSJDWxYugCNF2y9aqhS2jjFjcfuoJmcuHFQ5fQzHU33WboEtrJ0AW0se72Ow5dQjMrL7l66BKauekFVwxdQhNX3e7GQ5fQzLLLrxm6hAVhi6UkSZKaMFhKkiSpCYOlJEmSmjBYSpIkqQmDpSRJkpowWEqSJKkJg6UkSZKaMFhKkiSpCYOlJEmSmjBYSpIkqQmDpSRJkpowWEqSJKkJg6UkSZKaMFhKkiSpCYOlJEmSmjBYSpIkqQmDpSRJkpowWEqSJKkJg6UkSZKaMFhKkiSpCYOlJEmSmjBYSpIkqQmD5SySrE7y7iRvTXJxkguSvCjJqiTvSnJJknOSPGPkObdL8okkv+w//jvJXYc8D0mSpIVksFy/fYHLgAcCbwbeBvwn8ANgd+BDwPuS3CbJNsARwNXA3sBewHnAV/ttkiRJU89guX7/V1Wvq6ofAv8EXAisqaq3V9UZwBuAAA8G/l//eL+qOqmqTgOeB2wHPG62gyc5IMnxSY5fU1cvxPlIkiTNqxVDF7CInTTzoKoqyS+Ak0fWrUnyS+CWwL2AOwKXJRk9xjbAnWc7eFUdChwKsP2yHap59ZIkSQvMYLl+a8aWaz3rlvUf36VruRx3cfPKJEmSFiGDZRsnAn8CXFhVlwxciyRJ0iDsY9nGR4Hzgc8l2TvJHZM8rB9V7shwSZK0JBgsG6iqK4GHAWcC/wGcRjdq/GbALwcsTZIkacF4KXwWVbXPLOt2mWXdrUcenw/sN7+VSZIkLV62WEqSJKkJg6UkSZKaMFhKkiSpCYOlJEmSmjBYSpIkqQmDpSRJkpowWEqSJKkJg6UkSZKaMFhKkiSpCYOlJEmSmjBYSpIkqQmDpSRJkpowWEqSJKkJg6UkSZKaMFhKkiSpCYOlJEmSmjBYSpIkqQmDpSRJkppYMXQBAqqoNdcOXUUTay+ZjvPgV5cOXYFmsfykM4YuoZllN7vp0CU0UZdeNnQJ7dx6x6EraOanj7vV0CU0cfUta+gSmrndupsOXUI7p61/ky2WkiRJasJgKUmSpCYMlpIkSWrCYClJkqQmDJaSJElqwmApSZKkJgyWkiRJasJgKUmSpCYMlpIkSWrCYClJkqQmDJaSJElqwmApSZKkJgyWkiRJasJgKUmSpCYMlpIkSWrCYClJkqQmDJaSJElqwmApSZKkJgyWkiRJasJgKUmSpCYMlpIkSWrCYClJkqQmDJaSJElqwmApSZKkJgyWkiRJamLF0AUsVUkOAA4A2JptBq5GkiRpy9liOZCqOrSqdq+q3VeyauhyJEmStpjBUpIkSU0YLCVJktSEwXIeJXlhktOGrkOSJGkhGCzn1y2Auw9dhCRJ0kIwWM6jqnpdVWXoOiRJkhaCwVKSJElNGCwlSZLUhMFSkiRJTRgsJUmS1ITBUpIkSU0YLCVJktSEwVKSJElNGCwlSZLUhMFSkiRJTRgsJUmS1ITBUpIkSU0YLCVJktSEwVKSJElNGCwlSZLUhMFSkiRJTRgsJUmS1ITBUpIkSU0YLCVJktTEiqELEGT5MpZvt/3QZTSRG283dAlN1DZbD11CM7lu7dAlNHPdjtPxPgGodTV0CU2su8OOQ5fQzLIr1wxdQjO/9aULhi6hiXXbTc/v4mW/unLoEhaELZaSJElqwmApSZKkJgyWkiRJasJgKUmSpCYMlpIkSWrCYClJkqQmDJaSJElqwmApSZKkJgyWkiRJasJgKUmSpCYMlpIkSWrCYClJkqQmDJaSJElqwmApSZKkJgyWkiRJasJgKUmSpCYMlpIkSWrCYClJkqQmDJaSJElqwmApSZKkJqYmWCZZnaT6jz0HruWskVpuMWQtkiRJC2VqgmXvg8BtgBMARsLd+MeB/fZ9+uXTkqwYPVAfDl86sjwaXK9Ncl6Sw5M8PUnG6tgDeNL8nqokSdLiMm3B8sqq+nlVrRlZ91y6sDn68aGx590B2H8Ox58JrncCHg8cDbwH+GyS5TM7VdUFwMWbexKSJEmTaMXGd5l4l1TVzzeyzzuA1yX5SFVdsYH9rhw51k+Bbyc5BjgceCZd8JQkSVqSpq3FcnP9C7AGeMmmPrGq/gc4mU289J3kgCTHJzn+2nVXb+rLSpIkLTpLIVh+OMnlYx/3HtvnauA1wMuS7LgZr3EK3eXxOauqQ6tq96rafatlW2/GS0qSJC0uSyFYvgzYdezj9Fn2+zBwFl3A3FQBajOeJ0mSNDWWQh/Ln1fVGRvbqarWJXkl8J9J3r6Jr7EzcOZmVSdJkjQllkKL5ZxV1ReBo4A3zvU5SR4F7AJ8ar7qkiRJmgRLocXypkluPbbu8qq6fD37vxw4hm4wz7ht+mOtoJt26LH9/p8DPtKoXkmSpIm0FFos3wucN/bxyvXtXFXfpmt9XDXL5v36558JfAHYCzgQeGJVrW1btiRJ0mSZ6hbLqhq/I8749tV0A2/G1z8VeOrYun1a1iZJkjRtpq3F8oB+OqE9hiwiyf8BXxqyBkmSpIU2TS2W+wI36h//ZMhC6Pperuwfe2tHSZK0JExNsKyqnw1dw4yqOnvoGiRJkhbatF0KlyRJ0kAMlpIkSWrCYClJkqQmDJaSJElqwmApSZKkJgyWkiRJasJgKUmSpCYMlpIkSWrCYClJkqQmDJaSJElqwmApSZKkJgyWkiRJasJgKUmSpCYMlpIkSWrCYClJkqQmUlVD17DkJbkAOHsBXuoWwIUL8DrzbVrOAzyXxWpazmVazgM8l8VqWs5lWs4DFuZc7lBVO862wWC5hCQ5vqp2H7qOLTUt5wGey2I1LecyLecBnstiNS3nMi3nAcOfi5fCJUmS1ITBUpIkSU0YLJeWQ4cuoJFpOQ/wXBaraTmXaTkP8FwWq2k5l2k5Dxj4XOxjKUkDSnJ5VW3X+Jg7AQ+qqo9tyrY5Hnsf4Nqq+tYWlChpStliKUnTZyfgaZuxbS72AR60Bc+XNMUMlpK0CCTZJ8nqJJ9KclqSjyZJv+2sJG9JcnKS45LcpV9/WJInjxzj8v7hm4GHJvlukhePvdQNtiVZnuQfk3w7yUlJntcf68VJPtA/vneS7yfZGTgQeHH//IfO71dF0qRZMXQBkqRfux9wL+Bc4CjgwcA3+22/qqp7J3km8DbgcRs4ziuBl1bVbPvcYFuSA/pj75FkFXBUki8DbwdWJ3ki8NfA86rqlCSHAJdX1cFberKSpo8tlpK0eBxXVT+tqnXAd+kuW8/4+MjnvRq+5iOBZyb5LnAssANw176GZwMfBo6sqqMavqakKWWLpSQtHteMPF7LDX9H1yyPr6NvIEiyDNhqM14zwJ9X1f/Msu2uwOXAbTfjuJKWIFssJWkyPHXk89H947OA+/ePHw+s7B9fBtx4PccZ3/Y/wPOTrARIcrck2ya5CfAO4GHADiN9OTd0bElLnMFSkibDzZKcBLwImBmQ815g7yTfo7s8fkW//iRgbZLvzTJ4Z3zb+4BTgBOTfB94D11L6T8D76qqHwD7A29OckvgC8ATHbwjaTbOYylJi1ySs4Ddq+rCoWuRpA2xxVKSJElN2GIpSZKkJmyxlCRJUhMGS0mSJDVhsJQkSVITBktJkiQ1YbCUJElSE/8fRhU53wmoen8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x720 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "i = 2\n",
        "plot_attention(result['attention'][i], three_input_text[i], result['text'][i])"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "930d4775",
      "metadata": {
        "id": "930d4775"
      },
      "source": [
        "The short sentences often work well, but if the input is too long the model literally loses focus and stops providing reasonable predictions. There are two main reasons for this:\n",
        "\n",
        "The model was trained with teacher-forcing feeding the correct token at each step, regardless of the model's predictions. The model could be made more robust if it were sometimes fed its own predictions.\n",
        "\n",
        "The model only has access to its previous output through the RNN state. If the RNN state gets corrupted, there's no way for the model to recover. Transformers solve this by using self-attention in the encoder and decoder."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "bf7699fe",
      "metadata": {
        "id": "bf7699fe",
        "outputId": "67797c68-b2a1-4c8b-f2d6-7884c4204558"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Expected output:\n",
            " I learned nothing from the teacher.\n"
          ]
        }
      ],
      "source": [
        "long_input_text = tf.constant([inp[-1]])\n",
        "\n",
        "import textwrap\n",
        "print('Expected output:\\n', '\\n'.join(textwrap.wrap(targ[-1])))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "95712169",
      "metadata": {
        "id": "95712169",
        "outputId": "edbb5b18-4815-46ab-f2ff-67422435a4b1"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\Nhung\\AppData\\Local\\Temp/ipykernel_31496/2069917776.py:14: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
            "  ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
            "C:\\Users\\Nhung\\AppData\\Local\\Temp/ipykernel_31496/2069917776.py:15: UserWarning: FixedFormatter should only be used together with FixedLocator\n",
            "  ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnoAAAJKCAYAAAC70OihAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA3FklEQVR4nO3deZxkd13u8c8zSzJZCJgFCCgQRJRNtrCELRGQoAIKLiwBJUZiIHgjCMqiXvQq+1W2yxKuEHa4sqggmyKJgIGwCAGSEEgMJCSBLJI9mcnM9/5xTpNKp3umetJVp+rXn/fr1a9UnXOq6unOVPXTv3PO76SqkCRJUnvWDR1AkiRJk2HRkyRJapRFT5IkqVEWPUmSpEZZ9CRJkhpl0ZMkSWqURU/SzEryoiTv3M76byY5ZHqJ5lOS45P83tA5JE3fhqEDSFq7klw+cnd34Bpga3//93f0+Kq6yyRySVIrHNGTNJiq2nPhC/ge8OiRZe8aOt+kpDPxz99pvY6k2eUHgKRZt0uStye5rN9Ve+DCiiRnJXl4f/u+Sb6U5NIkP0jyN0s9WZJDkpyT5I+S/DDJeUkOH1m/a5JXJvle/zxvTLJbv+7UJI8a2XZDkguS3Ku/f/8k/5HkR0m+Nrpbud99+tdJPgdcCdx+Ua7Dk3x45P63k/z9yP2zk9yjv/2AJF9Mckn/3wes4HX2T3Jykuf295+a5Mz+5/tfSQ7b4f8RSXPDoidp1j0GeC9wM+CfgNcts92rgVdX1V7ATwP/bzvPeUvgpsCtgSOA/5PkJ/p1LwXuCNwDuEO/zZ/3694DPHHkeQ4FLqyqryS5NfDPwF8BewPPAT6QZL+R7Z8CHAncBPjuokwnAA9Osi7JrYBdgIMAktwe2BM4Ocne/eu8BtgH+Bvgn5Pss6PXSXJA/zqvq6pXJNmjf55fqqqbAA8Avrqdn5ukOWPRkzTrPltVH62qrcA7gLsvs90W4A5J9q2qy6vq89t5zi3AX1bVlqr6KHA58LNJQleQnlVVF1fVZcCLgSf0j3s38Jgku/f3n0RX/gCeDHy0z7qtqv4F+BLwyyOve1xVfbOqrq2qLaOBqupM4DK6gvkQ4BPAuUl+DjgY+ExVbQN+Bfh2Vb2jf573AKcBj97B69wZ+DTwP6vq2JFttwF3TbJbVZ1XVd/czs9N0pyx6EmadeeP3L4S2JRkqRPJjqAbiTut3535qCW2WXBRVV276Hn3BPajOynky/3u1x8BH++XU1XfAU4FHt2XvcfQlT+A2wK/ufC4/rEPAvYfeZ2zd/C9ngAcQlf0TgCOpyt5B/f3AW7FDUcDv0s38ri91zkM+D7w/oUFVXUF8HjgKOC8JP/cF0tJjbDoSWpCVX27qp4I3Bx4GfD+ftfkSlwIXAXcpapu1n/dtD9ZZMHC7ttfBU7pyx905eodI4+7WVXtUVUvHY25g9dfKHoP7m+fwA2L3rl0pXLUbehK3PZe50X99/fuJOt/vGHVJ6rqF+kK6WnAm3eQUdIcsehJakKSJyfZr9+9+aN+8baVPEf/2DcDf5vk5v3z3jrJoSObvRd4BPB0rhvNA3gn3UjfoUnWJ9nUn/jxkyuIcALwC8BuVXUO8BngkXTH4v1nv81HgTsmeVJ/Msjj6XbLfmQHz70F+E1gD+Dt/bGAt0jyq30hvoZuF/aKfmaSZptFT1IrHgl8s5+b79XAE6rqqp14nj8BvgN8PsmlwL8CP7uwsqrOA06kO3HhfSPLz6Yb5XsBcAHdCN9zWcHnbFWdTle2PtPfvxQ4E/hcf4wiVXUR8Cjgj4CLgD8GHlVVF47x/JuBxwG3AN5CN5fqs+lGCS+mGzl8+rh5Jc2+VO1oT4IkSZLmkSN6kiRJjbLoSZIkNcqiJ0mS1CiLniRJUqMsepIkSY2y6EmSJDXKoidJktQoi54kSVKjLHqSJEmNsuhJkiQ1yqInSZLUKIueJElSoyx6kiRJjbLoSZIkNcqiJ0mS1CiLniRJUqMsepIkSY2y6EmSJDXKoidJktQoi54kSVKjLHqSJEmNsuhJkiQ1yqInSZLUKIueJElSoyx6kiRJjbLoSZIkNcqiJ0mS1CiLniRJUqMsepIkSY2y6EmSJDXKoidJktQoi54kSVKjLHqSJEmNsuhJkiQ1yqInSZLUKIueJElSoyx6kiRJjbLoSZIkNcqiJ0mS1CiLniRJUqMsepIkSY2y6EmSJDXKoidJktQoi54kSVKjLHqSJEmNsuhJkiQ1yqInSZLUKIueJElSoyx6kiRJjbLoSZIkNcqiJ0mS1CiLniRJUqMsepIkSY2y6EmSJDXKoidJktQoi54kSVKjLHqSJEmNsuhJkiQ1yqInSZLUKIueJElSozYMHaA1SR63Ew/7WFVdtephJEnSmpaqGjpDU5JsW+FDCviZqjpzEnkkSdLa5YjeZNyyqn44zoZJLpt0GEmStDZ5jN7qexuwkt2w7wQunVAWSZK0hrnrVpIkqVGO6E1Akq1Jbj50DkmStLZZ9CYjQweQJEmy6EmSJDXKs24n57eSbPcki6p6+7TCSJKktceTMSagn0vvSro58pZTVbXXlCJJkqQ1yKI3AX3RG3suPUmSpEnwGL3JsD1LkqTBWfQmw7NuJyjJtn4Km6W+rkjytST/Y+ickiQNzZMxJmO7V8dIciDwV1X1yOlFasozgRcBHwK+0C+7H/BrwMuAnwJemqSq6rVDBJQkrV1J9l7pY6rq4olk8Ri9yUjyi8AjgC3A/62qM5PcEXgF8CjgXyx6OyfJPwL/VFV/t2j5EcBjqupXkxwF/EFV3WWQkJKkNas/Vn8lBauAO1bVmauexaK3+pL8DvBW4GJgb+AC4BjgTcAHgb+pqq8Pl3C+JbkcuEdVfWfR8jsAX6uqPZL8NPD1qtp9kJCSpDWrL3q/TtcDdrg58FHgrpMoeu66nYxnAS+oqpcm+S3gvcBzgXtV1RnDRmvCRXS7aV+5aPmvARf2t/cELpleJEmSfuy7wL9X1UXjbJzkTLo9gKvOojcZPw28r7/9fmAr8GxL3qr5C+DNSR4KnNQvuw/drvKn9fd/EThhgGySpDWuqg5Y4fZ3nVQWd91OwOJ59JJcBtx9EkOya1WSg4A/AH6uX3Qa8Jqq+vxwqSRJmi2O6E3OryRZ2HW4Djg0yQ9GN6iqD04/Vhuq6kTgxKFztCLJLYENVXXOouU/CWypqh8s/UhJ0nKSBPhtuuP1bk930sWZwN8D76opjLY5ojcB/YjejlRVrZ94mIYluRVwcxbNB1lVXxkm0fxK8q/A+6rqzYuWHwE8vqoeMUwySZpfST5Id/z414FT6E68uDNwV+BDVfXrE89g0dO8SXJP4J10u20XT05tgd4JSX4E3K+qvrVo+R2Bz1fViueEkqS1LMlhdLNtPK6qPrlo3aHAB4Ajq+rdk8zhlTEGkuThQ2eYY8cCZwMPphsKP2Dk6/YD5ppnG4Bdl1i+aZnlkqTtezLwssUlD6CqPkE3r+6TJx3CEb0pSnJr4HDgd4HbOvK0c5JcAdyzqk4fOksrknwKOL2qnr5o+ZuAn62qQwYJpqlLsgm4A92xRGdU1dUDR5LmUpJzgUdX1ZeXWX8g3eT/t5pkDk/GmLAk64FfBY6gm/7jZOCNdAdiaud8HbglYNFbPS8E/i3JzwP/1i97KHBPwNHnNSDJBuAldJcY3IXusIhrkrwWeGFVTWSOr52V5H7Aw1j6OF2vda1ZsA9w3nbWn0d3UYWJsuhNSJKfBX6P7mybK4B30xW9p1TVKUNma8ALgJcn+VO60ne9X0CTul5gy6rq8/2UNX8MPK5f/J/AM6rqa8Ml0xS9HHgicBTw2X7Zg+nK3zrgOQPluoEkz6HL+x3gXK5/qSl3U2lWbGT7kyBf228zUe66nYAkn6E7o+YDwDuq6oR++Ra6+fQsejfCorOaR/8BB0/GkHZKkvOB362qjy5a/it01+vef5hkN5TkbLpjn143dBZpOf3vqrcAVy6zye7A4ZP+neWI3mQcBPwf4Niq+ubQYRr0C0MHaFGSXYHD6E79L+CbwHuq6ppBg2labgosdfWeM4CbTTfKDu1Fd21QaZb9O92Vsna0zUQ5ojcB/fQfvwc8CTgLeDvwHrozRR3R08xJcmfg43S/QL/eL74b3fWCH1lVpw6VTdOR5PPAl6vq6EXL3wDco6oOGibZDSV5I3ByVb1+6CzSrLPoTVB/9tpv0p1l+yC641yeR7cb5L+HzDbvktwCOJrrjz69Ydau4DAvV5xI8i90uxeeUlWX9sv2opuvcNeqOnTIfJq8JA+hGyX7PrBwKcH7A7cCfqmqPrvcY6ctyQuBPwQ+SXeC2+LjdP9mgFjSTLLoTUCS2wBnj17aJMkduO7kjH2Af6uqXxoo4lxL8kC60acfcN1l0A6iO/vu0P7yaDNhXq44keRK4D6LDzVIcje6CZP3GCaZpqm/2szRXHcN6VOB11fVucOluqEk/7Wd1VVVzqepwSV59jjbTfoPE4veBCTZCuxfVT9cYt164FF0Bz3/6tTDNSDJiXS7F4+qqm39snV009bctaoeMGS+UfNyxYkkF9PN9/S5RcsfBPxjVe0zTDJJmk87+oOEbpqwXT0ZYz4tvizXj1XVVuAf+y/tnHsAT10oeQBVtS3J39BNCTJL5uWKEx8G3pzkaVy32+4gusv3/NNgqTR1/ajebejm0vuxqpr4QeM7oz+M44LRzwNpFlTVAUstT3J74K/pDu2a+Jy6XgJN8+gSusudLXYA8KPpRtmhLwBPX2L50cAXp5xle44Bvg18Bri6/zqBblLqPxwulqYlya2SHA+cA3wOOB749MjXzEiyMcnLk1xGd0zh7frlL0vyjEHDSctIsk+SVwGn0B1qdP+qesKkX9cRvcl5TpLLt7dBVf3ltMI05r3A3yX5Y+A/+mUPBF5Gd3bzLJmLK05U1Y+AX03yM4wcn1VV3xkulabsVcBWuhOcvgg8ErgF8JfAs4aLtaT/CTya7jqhoxeEPwn4E8CzcTUzkuwGPJtuQvqzgMdW1cem9voeo7f6+kkSv0U36/Vyqqp+fkqRmpJkF7qLQR/FdX+sbAHeAPxJVW0eKttSktyd7g1+j37RfwKv8IoTmiVJfgD8SlV9KcmlwIFVdXo/YfKfVdX9B474Y0nOoDvO+YR+VO/uVXVmf0WiL1TVzYZNKP342PEjgL+g+x31Z3QXUZhq8XJEb3IOXupkjFmR5J+AJ1fVpf3tZVXVY6YUayx9kTsmyfO5bjLKM6pqudnHB9UXusOGzrE9SV6zvfVeO3RN2A24sL99Md2updPpdjPN2h+ltwK+u8TyDczg7zUnI1+zvgHcFngN8Fq6Q2J+Irn+YfyTvmznzL0hGjEPw6QXcV3Oi4YMsrP6Yvf1HW44sDn5kL/bovsb6Xbhrmf2TnDRZJxG9//8LOCrwFH9pcaOpjsObpZ8E3gIXdZRvwV8eepptqOfjPxjdFceWfi8ehrwF0mcjLxtC4fB/AndXp3FQvc7YaJn3brrdgL6Xbe3nOURvXnWT0R9DPAwulGH651UNEu7xOf5ihP9z/nvgM9U1RuHzqPJSnIYsLGqjktyL7p/t/sA1wC/U1UTPztwXEkeTTeZ98vpjoP9C7pfqk+i2/38rwPGu555moy8PyzmhcAT6c683ji63uuIr0ySg8fZrqpOmGgOi97qS/I/6Y7Bmsldicvpf7Hfge4vjDOq6uqBIy0pyVuAx9Kdln4ui0ZQq+ovhsi1lHn6kF9KkrsAH6+qnxo6yzzqC9Oyquor08qyUkl2pytP36uqC3e0/bQlORR4AXBvuj/2vgL8ZVV9ctBgi8zTZORJXgY8HngJ8LfAn9Kd0fwEuuM03zRcOu0si94EJNmT7pf4RSPL7gQ8F9gT+GBVvXeofIsl2Qi8GHgm3dxZofsr/rXAC6tqy3YePnX95L6/NUt/tS9nnj7kl9L/RfoPVfUTQ2eZR/3ofnH9uTV//KHrCEn75mky8n6C36dX1cf7k1zuUVVnJHk68LCq+o2BI86VJEcCb1s4TKf/w/lbVXVtf38PuhMI/3ySOTxGbzLeQLdr7pkASfalm59sG3Ae8K4k66rq3cs/xVS9jG6o/ihg4XqWD6b7q24d8JyBci3nSuDsoUOM6WrgZkssv2m/biYscameAPvTHVv40eknur7+ZJHnV9UVc3biyOL5HjfSTa3zQuD5049zfXP8c50n8zQZ+S3oTr4BuJzrPrs+Tvd7QivzBuAfgIXDuE6km33hzP7+nnSfBRa9OXQQ8Psj958CbAbuVFWX9MPjz+T68z8N6Ul0UxWM/kI/I8kFwP9l9orey4FnJzlq2qep74R5+ZD/g0X3twEXAG+lK/xDuxvXHS+0+MSRmVVVS50Z+p0kl9DNBTe1ubSWMTc/137Kl9tX1YX9aNOy7/2q2mt6yXboGOBtdH/sb+2Xrae7OtKszU/4Pbozmr8HfAc4lO7kloOAqwbMNa8WXyVr2atmTZJFbzL2B84Yuf8LwAeq6pL+/tuA3516quXdlOvnXXAGS49GTd0SU8A8BHhkklPo5if6sRmbDmamP+STvA147nKX6pkVVfULS92eY//FdfMqDmbOfq5/AFzW337mkEHGMfLe+iHdZOR3AO7Ur57Vycg/RHeS2+eBVwPv6f9IvTXd3KWaQxa9ybgSGD326r7A+0buXw3sPtVE2/c14H/QTaMw6hi6aRZmweIpYD40SIoVGrnixMx9yCd5K91B1rv3IyTLqqqZ+cOkPxlnKUX33voO8L6qOnd6qZaWZO/Fi+j+EHwR3aTqM2PWf65V9baRu48F3gF8eNYmSIex3luPXZhLbZbeW1X1/JHb7++n13kgcHpVfWS4ZLoxLHqT8TXgcLrLoB0C7Md1l7+CbpLfwX8Jjfhj4KNJHs51uxfvTzeE/0uDpRpRVYcv3E5yZFUdu9R2SWZuGpAkj2fRVDAjH/JDjj7uR1c89gY20Y2SbuO6aWDuSpd31i5mvx/dMaTb6CYkhS5r6HYzPQ74yyQPrqqvDpLwOhdyw12MoTvG9PHTj7Nd8/RzvZJupHxLkg/QXW1golNUrNBcvreS/DVw9sJ0SlX1BeALSY5K8r+q6s+GTTiXfqU/VAO6/+eH9lehgSntMfOs2wnoz1T8GN2H/H7Au6vqiJH1rwd2Gy0vQ0pyG7rLtR3NyHVO6a4XuaGqvjdUtqUk+W/g96rqA4uWvxH4paq67TDJbijJK4A/pLso/FJTwQz6b6A/7ukedKXjnsDhVXVFv24Punn0vl5Vfz1YyEWSPA+4O3DEwhRG/VQgb6b7I+tVwNuB/arqYUPl7HP9Dl2pW9htv3Ds45l0c23OzHtrnn6u8ON/n4+lO8b44XQnur0HeGdVfWN7j52GOX1vfQ/4zb7gjS6/D/D+WfpsnQf9Wfc7UpM++96iNyH9dCqPAM4H/r6qto2sOxI4aQb+KgYgyVZg/8UTPCfZB/jhrE0BkeRhwAeBx1XVp/plx9JdhP2Qqjpze4+fpv4vt6Or6v1DZ1lKktPoDrj+PN30CacsWn8X4FNVdcsh8i0lyXnAQxdPNt1PTv2pqto/yT2Bfx166op5em/N0891sST70RWqo4Cfq6rB91bN6XvrauDOiz9Dk9weOKWqNg2TTDfGuh1vopVIct8k66vq1Kp6dVW9b7TkAVTVsQslL8m9+3nshrRwGZbF9mSGpgBZ0Je7I4D3J7lfkjfTfaDOVMnrrWN2jnO8gar6uf7M0D3pdtUvtj+zdTwpdFn3X2L5Lft1AJcyG4emzNN7a55+rj/WT/T+ULrPgDsyI1Mvzel763t0u+8XewhwzpSzzLWFLrCC7SfWBWbqDduIE+k+GC8Yc/tPc/15daZmZN6sAl7ST+67YD3dSSRfnXaucfQHCv8E3TEu5wEHV9VZw6Za0rHAk+kOvp9lHwDemuS5XP84zZfRjZ7Okg8Bf5fkj4Ev9svuQzftzkLW+wKnD5ANmNv31sz/XBekO8j1F+nmefw1ul3jf083cvaZAaMtZZ7eW28C/jbdpdAWjit/GN0US86jtzIz0wUseqsv3PCDfXt2mWSYHViYNyt0Z4SOnr22me6SQq+cdqilbGcy1x/SHeD87JETHAad2HVR1nXAYUl+ETiZG04FMyuT0D4d+N/AcVw3r9q1dMcRzdo8ikcBf0N3GbmFz7BrgbdwXdZT6S4cP5S5eW+NmIef64Lz6K4f/THgqcA/z+LZt725eW9V1f/uJ/h/Ddf9btoMvLqqXj5cspVJcirwMwPvwp+ZLuAxeqssyfFsZyLPZTypqs6bQJyx9FMBHFP9tVhnUZJPj7lpVdVDJxpmB+Yp62L9QeI/3d89Y+Hg8Vk0D1nn4b212Jz8XJ9Gd+zzj4bOMq55+Lku6LPeub97alVdPmSelUryTGCfGvC657PUBSx6kiRJjfJkDEmSpEZZ9CRJkhpl0Zuifv68uWDWyTDrZMxL1nnJCWadFLNOhlmXZ9Gbrrn5h4hZJ8WskzEvWeclJ5h1Usw6GWZdhkVPkiSpUZ51u4RdsmttYo9Vf94tXMNGdl3V59x8q9XPCbD1iitYv8fqPvdd9x133siVueCirey3z+peSer0r0/m57qlrmZjVvkqQhN6D6/2v9esm9zflZvranZZxZ9rbRvnEpUrN4nPgElZ81mzuk+3YEtdw8asctYJ/Rpf8/8GJmQSWS/jvy+sqv2WWueEyUvYxB7cb93Dh44xlu8+/f5DRxjbSb/3hqEjjO2Rt73v0BHGVtdu2fFGM2DdbrsNHWFs266+ZugI46vJlNKJyPzsRMq6CTW9CahtDthMxBy9t/51299/d7l18/OukyRJ0opY9CRJkhpl0ZMkSWqURU+SJKlRFj1JkqRGWfQkSZIaZdGTJElqlEVPkiSpURY9SZKkRln0JEmSGmXRkyRJapRFT5IkqVEWPUmSpEZZ9CRJkhrVbNFLclySjwydQ5IkaSgbhg4wQccAAUhyPPCNqnrmoIkkSZKmqNmiV1WXDJ1BkiRpSM0WvSTHAfsCFwIHAwcnObpffUBVnTVQNEmSpKlotuiNOAa4I3Aa8IJ+2QXDxZEkSZqO5oteVV2SZDNwZVWdv9x2SY4EjgTYxO7TiidJkjQxzZ51u1JVdWxVHVhVB25k16HjSJIk3WgWPUmSpEatlaK3GVg/dAhJkqRpWitF7yzgvklul2TfJGvl+5YkSWvYWik8r6Qb1TuF7ozb2wwbR5IkafKaPeu2qp46cvt04KDh0kiSJE3fWhnRkyRJWnMsepIkSY2y6EmSJDXKoidJktQoi54kSVKjLHqSJEmNsuhJkiQ1yqInSZLUKIueJElSoyx6kiRJjbLoSZIkNcqiJ0mS1CiLniRJUqMsepIkSY3aMHSAmZX56MC3f9/FQ0cY2/97wk2HjjC2dbe/zdARxnfRfw+dYDy32HfoBGNb//3zh44wtm1XXT10hLElGTrC2LLH7kNHGN/WrUMnGN+2GjrB2Gqefq6XL79qPtqMJEmSVsyiJ0mS1CiLniRJUqMsepIkSY2y6EmSJDXKoidJktQoi54kSVKjLHqSJEmNsuhJkiQ1yqInSZLUKIueJElSoyx6kiRJjbLoSZIkNcqiJ0mS1CiLniRJUqMsepIkSY1qruglOS7JR4bOIUmSNLQNQweYgGOADB1CkiRpaM0Vvaq6ZOgMkiRJs6DpXbdJHpLk80kuT3JJkpOS3HXojJIkSdPQ3IjegiQbgH8E/g44DNgI3AvYOmQuSZKkaWm26AF7ATcDPlxVZ/TLTltu4yRHAkcCbGL3iYeTJEmatOZ23S6oqouB44BPJPnnJM9OcpvtbH9sVR1YVQduZNep5ZQkSZqUZoseQFUdDtwP+HfgMcC3khw6bCpJkqTpaLroAVTV16rqZVV1CHA88DvDJpIkSZqOZotekgOSvDTJA5LcNskvAD8PnDJ0NkmSpGlo+WSMK4E7An8P7Av8AHgX8LIhQ0mSJE1Lc0Wvqp46cvdxQ+WQJEkaWrO7biVJktY6i54kSVKjLHqSJEmNsuhJkiQ1yqInSZLUKIueJElSoyx6kiRJjbLoSZIkNcqiJ0mS1CiLniRJUqMsepIkSY2y6EmSJDXKoidJktQoi54kSVKjNgwdYBYlYd0uG4eOMZ4fXjx0grG99nlPGDrC2H5w+Pz8DbTPyfsOHWEse5y7eegIY9v18r2GjjC2bN4ydITxbds2dIKx1Tz9XDUZW7cOnWBVzM9vM0mSJK2IRU+SJKlRFj1JkqRGWfQkSZIaZdGTJElqlEVPkiSpURY9SZKkRln0JEmSGmXRkyRJapRFT5IkqVEWPUmSpEZZ9CRJkhpl0ZMkSWqURU+SJKlRFj1JkqRGWfQkSZIaNfNFL8nxSV43dA5JkqR5M/NFT5IkSTtnTRa9JBuHziBJkjRp81L01iV5cZILk/wwySuTrANIskuSlyU5J8mVSb6Y5NCFByY5JEkl+eUkJyXZDBy67CtJkiQ1Yl6K3mHAtcADgGcCfwg8vl/3VuBg4EnAXYG3AR9OcvdFz/Ey4E+BnwO+MPnIkiRJw9owdIAxnVJVf97fPj3J04CHJTkJeCJwu6r6Xr/+dUkeDvw+8IyR53hRVX1yuRdIciRwJMCm7LHq34AkSdK0zUvRO3nR/XOBmwP3AgKckmR0/a7Avy16zJe29wJVdSxwLMBN1+1TNyasJEnSLJiXordl0f2i2+28rr99nyW2uWrR/SsmE02SJGk2zUvRW85/0o3o3bKqPj10GEmSpFky10Wvqk5P8i7guCR/BHwF2Bs4BDizqj44ZD5JkqQhzXXR6x0OvBB4OfCTwMXASYAjfJIkaU2b+aJXVYcsseypI7e3AC/qv5Z6/PF0u3clSZLWlHmZR0+SJEkrZNGTJElqlEVPkiSpURY9SZKkRln0JEmSGmXRkyRJapRFT5IkqVEWPUmSpEZZ9CRJkhpl0ZMkSWqURU+SJKlRFj1JkqRGWfQkSZIaZdGTJElq1IahA8yk9evInnsMnWIs2bTr0BHGttv5Vw8dYWzF7kNHGNumi64dOsJ4MnSAFdhWQycYWzbM0cf4ujn6R5A5GgdZP0dZr52TzyuA7DJ0gvFdtfyqOfrXIUmSpJWw6EmSJDXKoidJktQoi54kSVKjLHqSJEmNsuhJkiQ1yqInSZLUKIueJElSoyx6kiRJjbLoSZIkNcqiJ0mS1CiLniRJUqMsepIkSY2y6EmSJDXKoidJktQoi54kSVKjLHqSJEmNsuhJkiQ1ak0WvSQbkmToHJIkSZM0N0UvnT9K8u0k1yQ5J8lL+nUvTfKtJFclOSvJy5NsGnnsi5J8I8lTk5wBXAPsMdT3IkmSNA0bhg6wAi8Gng48G/h3YD/gnv26K4DfBb4P3Bl4I12Z+7ORxx8APAn4TWAzcPVUUkuSJA1kLopekj2BZwF/WFVv6Rd/BzgRoKr+18jmZyV5MfAcrl/0dgGeUlU/WOY1jgSOBNi0bs/V/QYkSZIGMBdFj26UblfgU0utTPIbwB8CdwD2BNb3X6POWa7kAVTVscCxADfduF/d+MiSJEnDmptj9JaT5P7Ae4FPAI+m2537p8DGRZteMeVokiRJg5qXEb1T6Y65exjw7UXrHgh8f3T3bZLbTjGbJEnSTJqLoldVlyV5NfCSJNfQnYyxD3Bv4HTg1kkOoztm71DgiYOFlSRJmhFzUfR6zwf+m+4Ei58EfgC8varekOQVwKuA3YBPAn8OvH6gnJIkSTNhbopeVW0DXtp/LV73fLoiOOoNI+tfBLxogvEkSZJmztyfjCFJkqSlWfQkSZIaZdGTJElqlEVPkiSpURY9SZKkRln0JEmSGmXRkyRJapRFT5IkqVEWPUmSpEZZ9CRJkhpl0ZMkSWqURU+SJKlRFj1JkqRGWfQkSZIatWHoALOotm5l248uGTrGeC65bOgEY1t33vlDRxjbz3xj96EjjC032XPoCGP53mG3GzrC2Ha77U8NHWFs+554wdARxpbNW4aOMLZtN9lt6AhNyjXXDh1hbNkyP1nZTmVxRE+SJKlRFj1JkqRGWfQkSZIaZdGTJElqlEVPkiSpURY9SZKkRln0JEmSGmXRkyRJapRFT5IkqVEWPUmSpEZZ9CRJkhpl0ZMkSWqURU+SJKlRFj1JkqRGWfQkSZIatcOil2TXcZZJkiRptowzonfimMskSZI0QzYstyLJLYFbA7sluSeQftVewO5TyLYiSY4HvlFVzxw6iyRJ0ixYtugBhwJPBX4S+N9cV/QuBV4w2ViSJEm6sZYtelX1NuBtSX69qj4wxUwrluQ44GDg4CRH94sPAG4DvAK4O3AJ8G7gT6pq8xA5JUmSpmmcY/R+LclNF+4kuW2ST00w0844hu64wbcC+/dfW4CPAf8J3BM4Angi8JKBMkqSJE3VOEXvs8AXkvxykqcB/wK8aqKpVqiqLgE2A1dW1flVdT7wDOBc4BlVdWpVfQR4HvDMJDc4xjDJkUm+lORLW+qaqeaXJEmahO0dowdAVb0pyTeBTwMXAvfsi9SsuxPw+araNrLss8AuwB2Ak0c3rqpjgWMB9lq3d00rpCRJ0qSMM4/eU4C3AL8NHAd8NMndJ5xr0ixykiSpeTsc0QN+HXhQVf0QeE+SDwFvA+4xyWA7YTOwfuT+qcBvJVk3Mqr3oH67M6YdTpIkadp2OKJXVb9WVT9cOK6tqk4C7jvxZCt3FnDfJLdLsi/weuBWwOuT3CnJrwAvBV5XVVcOmFOSJGkqxtl1e1CSU4DT+vt3Z8ZOxui9km607hTgAmAj8Et0Z9x+lW7383twDkBJkrRGjLPr9lV0kyf/E0BVfS3JQyYZamdU1enAQYsWnwXcb/ppJEmShjfO9CpU1dmLFm2dQBZJkiStonFG9M5O8gCgkmykm5z41MnGkiRJ0o01zojeUcDRwK2B79OdbfuMCWaSJEnSKhhnRO9nq+qw0QVJHgh8bjKRJEmStBrGGdF77ZjLJEmSNEOWHdFLchDwAGC/JM8eWbUX15+YWJIkSTNoe7tudwH27Le5ycjyS4HfmGQoSZIk3XjLFr2qOgE4IclxVfXdKWaSJEnSKhjnEmiWPEmSpDk01oTJkiRJmj/jXOv2geMskyRJ0mxxehVJkqRGOb2KJElSo5xeRZIkqVFOr7KUgtpWQ6cYS9ZtGzrC2OblZwrAugydYGzbfmKvoSOMZe9Trx06wtiu2meOdlpsmJ+stW2OPq/Wz8/PlQ3zc17ltXttGjrC2HY564KhI6yKca51e1ySG/yGrqqHTiCPJEmSVsk4Re85I7c3Ab8OzM+f5pIkSWvUDoteVX150aLPJTlpQnkkSZK0SnZY9JLsPXJ3HXBv4KYTSyRJkqRVMc6u2y8DBYRul+1/AUdMMpQkSZJuvHF23R4wjSCSJElaXePsut0EPAN4EN3I3meAN1bV1RPOJkmSpBthnF23bwcu47rLnj0JeAfwm5MKJUmSpBtvnKJ316q688j9Tyc5ZVKBJEmStDrGmU77K0nuv3Anyf2AL00ukiRJklbDOCN69wb+I8n3+vu3Ab6V5OtAVdXPTyydJEmSdto4Re+RE08hSZKkVTdO0furqnrK6IIk71i8TJIkSbNlnGP07jJ6J8kGut25kiRJmmHLFr0kz09yGfDzSS5Ncll//wfAP04toSRJknbKskWvql5SVTcBXlFVe1XVTfqvfarq+VPMKEmSpJ0wzq7bjyV5yOKvlb5QknVJ3pTkoiSV5KwkH1nhcxyf5HVjbvuiJN9YaU5JkqRWjHMyxnNHbm8C7gt8GXjoCl/rl4HDgUOAM4GrgKzwOR4HbFnhYyRJktakHRa9qnr06P0kPwW8aide6w7AeVX1Hzvx2IUsF+/sYyVJktaacXbdLnYOcKeVPCDJccDfArcZ2W173Oiu23637OuTvDjJhUl+mOSVSdYt2uZ1I/cfl+TkJFcluTjJCUlusei1n5DkjP5kkn9Isu9OfM+SJElzZ4cjekleC1R/dx1wD+ArK3ydY4DvAr8L3AfYCrxiie0OA14NPKB/nXfT7SZ+zxK5bgm8F3g+8AFgT+D+iza7HfB44LHAHv32fw38/grzS5IkzZ1xjtEbva7ttcB7qupzK3mRqrqkn5pla1WdD5AseXjeKVX15/3t05M8DXgYSxQ94FbARuD9VfXdftniky82AE+tqkv61zyW7jjBG0hyJHAkwCZ2H/dbkyRJmlnjFL330R1fB/Cdqrp6gnlOXnT/XODmy2z7NeBfgW8k+WR/+/1VdcHINt9dKHk7er6qOhY4FmCv7F1LbSNJkjRPtjdh8oYkL6c7Ju9twNuBs5O8PMnGCeVZfEZtLZexqrYCj+i/TgaOAL6d5O4783ySJEmt2V7peQWwN3BAVd27qu4F/DRwM+CVU8i2Q9U5sar+gu7Yv3PpjsmTJEla87a36/ZRwB2r6se7Mavq0iRPB06jO8FiMEnuDzwc+ATdZdnuCfwUcMqQuSRJkmbF9opejZa8kYVbk8zCMWyXAA8E/oBulPFs4H9V1TuHDCVJkjQrskSX61Yk/wB8sKrevmj5k4HfqqrHTD7eMPbK3nW/9Y8YOsZYsm6lFxcZTm2bhb8PxrN+rz2HjjC2+sn9h44wlisP2GvoCGO7ap/1Q0cY275fnKN55DfPz4WNtu2529ARxrdhfg4937rr/Ly3djnrgh1vNCM+fvarv1xVBy61bnsjekcDH0zyu3Rz2QEcCOxGNy+dJEmSZtiyRa+qvg/cL8lDgbv0iz9aVZ+aSjJJkiTdKONc6/bfgH+bQhZJkiStovnZsS9JkqQVsehJkiQ1yqInSZLUKIueJElSoyx6kiRJjbLoSZIkNcqiJ0mS1CiLniRJUqMsepIkSY2y6EmSJDXKoidJktSoHV7rds3atnXoBGOpbUMnaNPWH10ydITxXXr50AnGsvt3Ng4dYWx73mK/oSOMrXaZn5/rllvsNXSEsV14992HjjC2Pc+bj99XABfdaf3QEcZ2u4v3GDrCqnBET5IkqVEWPUmSpEZZ9CRJkhpl0ZMkSWqURU+SJKlRFj1JkqRGWfQkSZIaZdGTJElqlEVPkiSpURY9SZKkRln0JEmSGmXRkyRJapRFT5IkqVEWPUmSpEZZ9CRJkhpl0ZMkSWrUTBe9JIckqST7Dp1FkiRp3sxU0UtyfJLXzcvzSpIkzbKZKnqSJElaPTNT9JIcBxwMHN3vri3gdv3quyf5QpIrk3wpyb1GHrdPkvckOSfJVUm+meTw7T1vkoXnlSRJatbMFD3gGOBE4K3A/v3X2f26lwDPA+4FXAS8K0n6dZuArwCPAu4CvBp4U5KHjfG8kiRJzdowdIAFVXVJks3AlVV1PkCSn+tX/1lVfbpf9pfAZ4FbA+dU1feBV4w81bFJHgo8EfjUUs+7lCRHAkcCbGL3Vf7uJEmSpm+WRvS25+SR2+f2/705QJL1SV6Y5OQkFyW5HHgccJuVvEBVHVtVB1bVgRvZdXVSS5IkDWhmRvR2YMvI7er/u1BSnwP8Ed0u2q8DlwMvpi+CkiRJa9WsFb3NwPoVPuZBwIer6h0A/bF7dwR+dCOfV5Ikaa7N2q7bs4D7JrldP0nyOPlOBx6W5EH9MX2vAw7Y3vMmmbXvW5IkadXNWuF5Jd3o2ynABYx3nN1fAScBHwP+HbgCeNcqPK8kSdJcm6ldt1V1OnDQosXHLdrmLCAj9/+b7uSLlT6vJElS02ZtRE+SJEmrxKInSZLUKIueJElSoyx6kiRJjbLoSZIkNcqiJ0mS1CiLniRJUqMsepIkSY2y6EmSJDXKoidJktQoi54kSVKjLHqSJEmNsuhJkiQ1yqInSZLUqA1DB5B0I9W2oROMJbvsMnSE8a2bn7+Bc/XmoSOMbcNl1wwdYWy1bvehI4xt6y4ZOsLYUkMnGF9tmJ/Pge1p47uQJEnSDVj0JEmSGmXRkyRJapRFT5IkqVEWPUmSpEZZ9CRJkhpl0ZMkSWqURU+SJKlRFj1JkqRGWfQkSZIaZdGTJElqlEVPkiSpURY9SZKkRln0JEmSGmXRkyRJapRFT5IkqVHNFr0kxyX5yNA5JEmShrJh6AATdAwQgCTHA9+oqmcOmkiSJGmKmi16VXXJ0BkkSZKG1GzRS3IcsC9wIXAwcHCSo/vVB1TVWQNFkyRJmopmi96IY4A7AqcBL+iXXbB4oyRHAkcCbGL3qYWTJEmalOaLXlVdkmQzcGVVnb+d7Y4FjgXYK3vXtPJJkiRNSrNn3UqSJK11Fj1JkqRGrZWitxlYP3QISZKkaVorRe8s4L5Jbpdk3yRr5fuWJElr2FopPK+kG9U7he6M29sMG0eSJGnymj3rtqqeOnL7dOCg4dJIkiRN31oZ0ZMkSVpzLHqSJEmNsuhJkiQ1yqInSZLUKIueJElSoyx6kiRJjbLoSZIkNcqiJ0mS1CiLniRJUqMsepIkSY2y6EmSJDXKoidJktQoi54kSVKjLHqSJEmN2jB0AN1I69YPnWB8tW3oBOOrGjpBc7L3zYaOMLbatMvQEcaWqzcPHWFs1+61aegIY9t4+fx8Bqy7dn6y3uR78/N7YOtN5uff6/Y4oidJktQoi54kSVKjLHqSJEmNsuhJkiQ1yqInSZLUKIueJElSoyx6kiRJjbLoSZIkNcqiJ0mS1CiLniRJUqMsepIkSY2y6EmSJDXKoidJktQoi54kSVKjLHqSJEmNsuhJkiQ1yqInSZLUKIueJElSoyx6kiRJjbLoSZIkNWrD0AFmRZIjgSMBNrH7wGkkSZJuPEf0elV1bFUdWFUHbmTXoeNIkiTdaBY9SZKkRln0JEmSGmXRkyRJatSaKnpJnpnktKFzSJIkTcOaKnrAvsDPDh1CkiRpGtZU0auqF1VVhs4hSZI0DWuq6EmSJK0lFj1JkqRGWfQkSZIaZdGTJElqlEVPkiSpURY9SZKkRln0JEmSGmXRkyRJapRFT5IkqVEWPUmSpEZZ9CRJkhpl0ZMkSWqURU+SJKlRFj1JkqRGbRg6wCzKunWs232PoWOMZd1eNxk6wvh22Th0grHVrrsMHWF8G+fjbbxtfYaOMLZav37oCGNbf/XmoSOMbZdzLh46wtj2uXzPoSOMbdvu8/PZuuUm85N1w4+uGjrCqnBET5IkqVEWPUmSpEZZ9CRJkhpl0ZMkSWqURU+SJKlRFj1JkqRGWfQkSZIaZdGTJElqlEVPkiSpURY9SZKkRln0JEmSGmXRkyRJapRFT5IkqVEWPUmSpEZZ9CRJkhpl0ZMkSWrUxIpekuOTVP91/0m9zphZzhrJsu+QWSRJkqZl0iN6bwX2B74MMFK2Fn8d1a8/pL9/WpINo0/Ul7XnjNwfLZKbk5yX5ONJnpwki3LcB/j1yX6rkiRJs2XSRe/Kqjq/qraMLHsaXfkb/XrbosfdFjhijOdfKJK3Bx4DnAi8CfhQkvULG1XVBcDFO/tNSJIkzaMNO95k1f2oqs7fwTavAV6U5J1VdcV2trty5LnOAb6Y5PPAx4HfpiuCkiRJa9KsnozxWmAL8OyVPrCqPgF8nRXuqk1yZJIvJfnS5rp6pS8rSZI0c4Yoeu9Icvmir7st2uZq4M+A5ybZbyde4xS63bljq6pjq+rAqjpwl2zaiZeUJEmaLUMUvecC91j09a0ltnsHcBZd4VupALUTj5MkSWrGEMfonV9V39nRRlW1LcnzgH9I8uoVvsadgTN3Kp0kSVIjZvUYPQCq6qPA54C/HvcxSQ4F7gq8f1K5JEmS5sEQI3o3S3LLRcsur6rLl9n+j4HP052csdju/XNtoJtm5Zf77f8ReOcq5ZUkSZpLQ4zovRk4b9HX85bbuKq+SDc6t+sSqw/vH38m8GHgIOAo4LFVtXV1Y0uSJM2XqY7oVdXiK1YsXn883YkUi5c/Hnj8omWHrGY2SZKk1kx6RO/IfvqU+0z4dbYryTeBjw2ZQZIkadomOaJ3GLBbf/vsCb7OOH4Z2Njf9lJokiRpTZhY0auq70/quVeqqr47dAZJkqRpm+npVSRJkrTzLHqSJEmNsuhJkiQ1yqInSZLUKIueJElSoyx6kiRJjbLoSZIkNcqiJ0mS1CiLniRJUqMsepIkSY2y6EmSJDUqVTV0hpmT5AJgEtfH3Re4cALPOwlmnQyzTsa8ZJ2XnGDWSTHrZKz1rLetqv2WWmHRm6IkX6qqA4fOMQ6zToZZJ2Ness5LTjDrpJh1Msy6PHfdSpIkNcqiJ0mS1CiL3nQdO3SAFTDrZJh1MuYl67zkBLNOilknw6zL8Bg9SRqR5PKq2nOVn/N2wAOq6t0rWTfmcx8CbK6q/7gRESU1yhE9SZq82wFP2ol14zgEeMCNeLykhln0JGkJSQ5JcnyS9yc5Lcm7kqRfd1aSlyf5epKTktyhX35ckt8YeY7L+5svBR6c5KtJnrXopa63Lsn6JK9I8sUkJyf5/f65npXkLf3tuyX5RpI7A0cBz+of/+DJ/lQkzZsNQweQpBl2T+AuwLnA54AHAp/t111SVXdL8tvAq4BHbed5ngc8p6qW2uZ665Ic2T/3fZLsCnwuySeBVwPHJ3ks8ELg96vqlCRvBC6vqlfe2G9WUnsc0ZOk5Z1UVedU1Tbgq3S7WRe8Z+S/B63iaz4C+O0kXwW+AOwD/Eyf4anAO4ATqupzq/iakhrliJ4kLe+akdtbuf5nZi1x+1r6P6CTrAN22YnXDPAHVfWJJdb9DHA5cKudeF5Ja5AjepK0cx4/8t8T+9tnAffubz8G2Njfvgy4yTLPs3jdJ4CnJ9kIkOSOSfZIclPgNcBDgH1GjgXc3nNLWuMsepK0c34iycnAMcDCCRZvBg5O8jW63blX9MtPBrYm+doSJ2MsXvd/gVOAryT5BvAmupHEvwX+T1WdDhwBvDTJzYEPA4/1ZAxJS3EePUlaoSRnAQdW1bxcRF3SGuWIniRJUqMc0ZMkSWqUI3qSJEmNsuhJkiQ1yqInSZLUKIueJElSoyx6kiRJjfr/HVAe84Rz3aUAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x720 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "result = translator.tf_translate(long_input_text)\n",
        "\n",
        "i = 0\n",
        "plot_attention(result['attention'][i], long_input_text[i], result['text'][i])\n",
        "_ = plt.suptitle('This never works')"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "c7d92daf",
      "metadata": {
        "id": "c7d92daf"
      },
      "source": [
        "### Export"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2fb63bbc",
      "metadata": {
        "id": "2fb63bbc",
        "outputId": "d8456ff8-9745-46ac-834b-01bc30652bde"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:absl:Found untraced functions such as encoder_2_layer_call_fn, encoder_2_layer_call_and_return_conditional_losses, decoder_2_layer_call_fn, decoder_2_layer_call_and_return_conditional_losses, embedding_4_layer_call_fn while saving (showing 5 of 60). These functions will not be directly callable after loading.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: translator\\assets\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: translator\\assets\n"
          ]
        }
      ],
      "source": [
        "tf.saved_model.save(translator, 'translator',\n",
        "                    signatures={'serving_default': translator.tf_translate})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3479c15f",
      "metadata": {
        "id": "3479c15f"
      },
      "outputs": [],
      "source": [
        "reloaded = tf.saved_model.load('translator')\n",
        "result = reloaded.tf_translate(three_input_text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "31afe20d",
      "metadata": {
        "id": "31afe20d",
        "outputId": "6b376cf1-ad8e-4cc5-c0e2-20e6a17a27c0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "he is he safe .\n",
            "the money is there my hair against school .\n",
            "not dont see saying me tired .\n",
            "\n",
            "Wall time: 576 ms\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "result = reloaded.tf_translate(three_input_text)\n",
        "\n",
        "for tr in result['text']:\n",
        "  print(tr.numpy().decode())\n",
        "\n",
        "print()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3847b6b8",
      "metadata": {
        "id": "3847b6b8"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.11"
    },
    "colab": {
      "name": "Seq2Seq Language Translation model, word model with attention",
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}