LSTM.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "name": "LSTM.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4VAOO44sUzv3"
      },
      "source": [
        "### Notebook part 2\n",
        "\n",
        "# Seq2Seq LSTM Modeling\n",
        "_By **Avi Patel**_\n",
        "\n",
        "## Overview\n",
        "This project seeks to create a model that creates one line abstractive summary of a given news article. This model will help to get a quick idea on what the article is all about without having to read all whole news.\n",
        "\n",
        "\n",
        "## Business Problem\n",
        "News/Media/Blogging Companies always wants a catchy  headlines for the articles they post so that they can get maximum number of clicks. A machine learning algorithm can be  used to create a headline based on the context of the article.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wgDhlFmpGWOX"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "from keras.preprocessing.text import Tokenizer \n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "import os\n",
        "import json\n",
        "\n",
        "import tensorflow as tf"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TW6mLL1dgJgm",
        "outputId": "10187724-255b-4f85-981c-1cbfd1201115"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ErZ79bM_GgUG"
      },
      "source": [
        "df = pd.read_csv('/content/drive/MyDrive/cleaned_news_summary.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "PkGpgGypIMLk",
        "outputId": "48160428-0ea3-4230-cd1d-7445f54ebdf0"
      },
      "source": [
        "df.replace('_START_','', regex=True, inplace=True)\n",
        "df.replace('_END_','', regex=True, inplace=True)\n",
        "\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>summary</th>\n",
              "      <th>cleaned_text</th>\n",
              "      <th>cleaned_summary</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Saurav Kant, an alumnus of upGrad and IIIT-B's...</td>\n",
              "      <td>upGrad learner switches to career in ML &amp; Al w...</td>\n",
              "      <td>saurav kant an alumnus of upgrad and iiit-b pg...</td>\n",
              "      <td>upgrad learner switches to career in ml al wit...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Kunal Shah's credit card bill payment platform...</td>\n",
              "      <td>Delhi techie wins free food from Swiggy for on...</td>\n",
              "      <td>kunal shah credit card bill payment platform c...</td>\n",
              "      <td>delhi techie wins free food from swiggy for on...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>New Zealand defeated India by 8 wickets in the...</td>\n",
              "      <td>New Zealand end Rohit Sharma-led India's 12-ma...</td>\n",
              "      <td>new zealand defeated india by wickets in the f...</td>\n",
              "      <td>new zealand end rohit sharma-led india 12-matc...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>With Aegon Life iTerm Insurance plan, customer...</td>\n",
              "      <td>Aegon life iTerm insurance plan helps customer...</td>\n",
              "      <td>with aegon life iterm insurance plan customers...</td>\n",
              "      <td>aegon life iterm insurance plan helps customer...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Speaking about the sexual harassment allegatio...</td>\n",
              "      <td>Have known Hirani for yrs, what if MeToo claim...</td>\n",
              "      <td>speaking about the sexual harassment allegatio...</td>\n",
              "      <td>have known hirani for yrs what if metoo claims...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  ...                                    cleaned_summary\n",
              "0  Saurav Kant, an alumnus of upGrad and IIIT-B's...  ...  upgrad learner switches to career in ml al wit...\n",
              "1  Kunal Shah's credit card bill payment platform...  ...  delhi techie wins free food from swiggy for on...\n",
              "2  New Zealand defeated India by 8 wickets in the...  ...  new zealand end rohit sharma-led india 12-matc...\n",
              "3  With Aegon Life iTerm Insurance plan, customer...  ...  aegon life iterm insurance plan helps customer...\n",
              "4  Speaking about the sexual harassment allegatio...  ...  have known hirani for yrs what if metoo claims...\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jkdwH9-tJebc"
      },
      "source": [
        "**Drop NaN Rows**\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kiyLk5voJA5y"
      },
      "source": [
        "df.dropna(axis=0, inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mzmi9vciSNJq",
        "outputId": "05416fb9-bc79-42e4-989a-9637bd51d41c"
      },
      "source": [
        "df.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(102797, 4)"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HrxY9XmsJki0"
      },
      "source": [
        "**Count total words in Summary and Text**\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0vl8boxeHt10"
      },
      "source": [
        "text_count = []\n",
        "summary_count = []"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FpZC9sOsH1RU"
      },
      "source": [
        "for txt in df['cleaned_text']:\n",
        "    text_count.append(len(txt.split()))\n",
        "for smry in df['cleaned_summary']:\n",
        "    summary_count.append(len(smry.split()))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Byl06eNiJKtr"
      },
      "source": [
        "df['text_count'] = text_count\n",
        "df['summary_count'] = summary_count"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        },
        "id": "eNGPC42iJOaS",
        "outputId": "9bb100a0-94a6-4fe3-eafe-7633f8d8bf59"
      },
      "source": [
        "df.hist(bins = 5)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEICAYAAACqMQjAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAeq0lEQVR4nO3dfZRdVZnn8e/PhJeIQALYZUzSBjWjK5ARIQNxfJlqXkKITodZCx2QloCRrBnBxjG2hHkLjeIKvRqRIDIdJZJgJNL4kiwJxgjccZzpBIIiISCmCMEkhgTICxSIGn3mj7NLDrfuTlXqVt17q/L7rHVXnfOcfc7Zu7LvffY+5+SWIgIzM7NaXtfsCpiZWetykjAzsywnCTMzy3KSMDOzLCcJMzPLcpIwM7MsJwkzM8tykjAzazGSKpI+0ex6gJNE00jaLOnMVjnOQJA0XlJIGt7sutjBzX2w75wkzKxXJF0paZukFyU9IekMSbdJ+kKpTLukraX1zZL+TtIjkl6SdKukNkn3pOP8WNKoVLZrUHGJpC2Sdkv6T5L+Tdp/j6SvlI79Nkn3SXpe0nOSlkoaWXXuKyU9AryU6vGdqjYtkHRjD+0+RtI3JP0m1en7pW2XSuqQtEvSCklvrmrL8FLZP88OJF0s6aeS/jEd8ylJ56Rt1wLvB74iqbPc5qaICL8a/AJuB/4E/BboBD4HTAH+H7AH+AXQnsr+W+A5YFxafxewG3hnreP0cN73lc6xBbg4xY8GlgDPAk8D/x14Xdp2NfDN0jHGAwEMT+sV4PPA/wVeBH4EHJe2/TqV7Uyv9zT7d+9Xn/vsO1KfeXOpH7wNuA34QqlcO7C1tL4ZWAO0AWOAncDPgHcDhwP3AfOq+tb/StumAq8A3wf+orT/v0vl3w6cBRwGvBH4CfDlqnM/DIwDRgCjgZeAkWn78HS8U3po+93At4FRwCGl85+e3psnpzrcBPykqi3DS8epAJ9IyxcDfwAuBYYB/xn4DaDqss1+Nb0CB+srdeAz0/IY4HlgOsXs7qy0/sa0/dr0ZhoBrAcur3WcHs73lvQhfkHq6McCJ6VtS4DlwJGpc/8KmJW2XU3PSeJJ4F+l+lWA+bXK+jV4X+kDeSdwJnBIKX4bPSeJC0vr3wFuKa1/Cvh+VX8ZU9r+PPAfq/b/dKaO5wI/rzr3x6vK3ANcmpY/BDzWQ7tHUwzERtXYdivwD6X1N6QP/vG1+j7dk0RHadvrU/k3VZdt9suXm1rD3wArI2JlRPwpIlYD6yiSBhQf1EcDDwDbgJv7cI6PAj+OiDsi4g8R8XxEPCxpGHA+cFVEvBgRm4HrgY8dwLG/ERG/iojfAncCJ/WhftbCIqID+DRFX9wpaVnXpZVe2FFa/m2N9Tf0pXy6bLUsXQJ7AfgmcFzVsbZUrS+meL+Rft7eQ93HAbsiYneNbW+mmHkDEBGdFEltTA/H7PJMad+X02L176LpnCRaw1uAD6drrnsk7aG4NDQaICL+QDFiOxG4PtJQ4wCNoxjxVzuOYmbxdCn2NL3v6FDq7MDLtGBHt/pFxLci4n0U/TWA6ygu37y+VOxNDazSF1M9JkXEURQf+qoqU/1e+T7wryWdSDGTWNrDObYAx5TvdZT8huJ3AYCkIyhm6Nsofi/Q999Ny3w9t5NE85Q7wRbg9ogYWXodERHzASSNAeYB3wCul3RY5jj7s4XiGnK15yimyG8pxf6SoqNDfR8CLdPRrT6S3iHp9NT3XqEY0f+J4pr/9HRz900Us41GOZLiXtfe9B75u552iIhXgLuAbwEPRMSveyi/neIS1VcljZJ0iKQPpM13AJdIOin9Xr4IrI2IzRHxLMV76G8kDZP0cWq//3J2AG89gPIDxkmiecqd4JvAv5d0dupQh6enRMZKEsUs4lZgFrCd4kZxrePsz1LgTEkfkTRc0rGSToqIP1JcIrpW0pGS3gJ8JtUJig+BD0j6S0lHA1cdQBufpfggaYnObnU5DJhPMah4huJG8lUUl2t+QXH9/0cUN3gb5e8pbhrvpbi5/N1e7rcYmETPl5q6fIxiIPVLivsynwaIiB8D/4PiPsl2iiRwfmm/SykS1/PACRQPjfTWjcB56cmnBQewX/9r9k2Rg/UFzKB4+mcP8FngNOB/A7soPlzvphjRX0HxJjw07ffmtP39tY7TwznfD6wFXqCYWcxM8VEUSeHZFP+fpKeb0vab0/E7KDp+9Y3rT5TKXgz8tLR+TTruHmBKs3/vfvmV3lcvA0c1uy6D4dX1uJWZ2ZAn6XXAlygSxMebXZ/BwP8L0cwOCunG8g6KBzOmVW3rzOx2TkT8n4GuWyvzTGIIkXQh8E81Nj0dESc0uj5mNvg5SZiZWdaQu9x03HHHxfjx47vFX3rpJY444ojGV6hBhnL7Gt22hx566LmIeGPDTlinXJ9vNYOtjx5s9c31+yGXJMaPH8+6deu6xSuVCu3t7Y2vUIMM5fY1um2Snu65VOvI9flWM9j66MFW31y/9/+TMDOzLCcJMzPLcpIwM7MsJwkzM8tykjAzsywnCTMzy3KSMDOzLCcJMzPLcpIwM7OsIfc/rnPWb9vLxXPvbsi5Ns//YEPOY3YwGt+g9/Ft0wbPV3IMJM8kzMwsq8ckIWmRpJ2SHi3FjpG0WtLG9HNUikvSAkkdkh6RdHJpn5mp/EZJM0vxUyStT/ssSH+uM3sOMzNrnN7MJG6j6g90AHOBeyNiAnBvWgc4B5iQXrOBW6D4wAfmUfyJzlOBeaUP/Vso/iRm137TejiHmZk1SI9JIiJ+QvF3l8tmUPwxcdLPc0vxJVFYA4yUNBo4G1gdEbsiYjewGpiWth0VEWui+MMWS6qOVescZk0jaXOa+T4saV2KDfjM2qxZ+nrjui0itqflZ4C2tDwG2FIqtzXF9hffWiO+v3N0I2k2xcyFtrY2KpVK9wqPgDmT9vXUrn5R6/wDrbOzsynnbYQWbNtfRcRzpfWuWe98SXPT+pW8dmZ9GsWs+bTSzHoyEMBDklakAVTXzHotsJJiZn1PY5pl1l3dTzdFREga0D9v19M5ImIhsBBg8uTJUes71W9aupzr1zfmYa7NF3Y//0AbbN99fyAGQdtmAO1peTFQoUgSf55ZA2skdc2s20kzawBJXTPrCmlmneJdM2snCWuavn5q7pA0OiK2p06/M8W3AeNK5cam2DZefRN1xSspPrZG+f2dw6yZAvhRGrT8UxqgNGJm/Rq9mT23mv6aETbqikALzmD3a6Dq29cksQKYCcxPP5eX4pdLWkYxvd6bPuRXAV8s3ayeClwVEbskvSBpCsX0+iLgph7OYdZM74uIbZL+Algt6ZfljY2YWafz9Dh7bjX9NSNs1P93um3aEa0+g32NgZpx9+YR2DuAfwHeIWmrpFkUH9xnSdoInJnWobiGugnoAL4GfBIgTas/DzyYXtd0TbVTma+nfZ7k1al17hxmTRMR29LPncD3KJ7W25FmuxzAzDoXz82szZqix5lERFyQ2XRGjbIBXJY5ziJgUY34OuDEGvHna53DrFkkHQG8LiJeTMtTgWtozMzarCkOmq/lMOsHbcD30lOpw4FvRcQPJT0I3Jlm2U8DH0nlVwLTKWbJLwOXQDGzltQ1s4buM+vbgBEUs2rftLamcpIw66WI2AS8q0a85qy3P2fWZs3i724yM7MsJwkzM8tykjAzsywnCTMzy/KNazOzGvyHygqeSZiZWZaThJmZZTlJmJlZlpOEmZllOUmYmVmWk4SZmWU5SZiZWZaThJmZZTlJmJlZlpOEmZllOUmYmVmWk4SZmWU5SZiZWZaThJmZZTlJmJlZlpOEmZllOUmYmVmWk4SZmWU5SZiZWZaThJmZZTlJmJlZlpOEmZllOUmYmVmWk4SZmWXVlSQk/RdJGyQ9KukOSYdLOl7SWkkdkr4t6dBU9rC03pG2jy8d56oUf0LS2aX4tBTrkDS3nrqamdmB63OSkDQG+FtgckScCAwDzgeuA26IiLcDu4FZaZdZwO4UvyGVQ9LEtN8JwDTgq5KGSRoG3AycA0wELkhlzcysQeq93DQcGCFpOPB6YDtwOnBX2r4YODctz0jrpO1nSFKKL4uI30XEU0AHcGp6dUTEpoj4PbAslTVrqjSI+bmkH6R1z55tyOpzkoiIbcA/Ar+mSA57gYeAPRGxLxXbCoxJy2OALWnffan8seV41T65uFmzXQE8Xlr37NmGrOF93VHSKIqR/fHAHuCfKTp8w0maDcwGaGtro1KpdCvTNgLmTNrXLT4Qap1/oHV2djblvI3QSm2TNBb4IHAt8Jk0Gz4d+Ggqshi4GriF4v1xdYrfBXylevYMPCWpa/YMafacztU1e35sgJtlltXnJAGcCTwVEc8CSPou8F5gpKThabYwFtiWym8DxgFb0+Wpo4HnS/Eu5X1y8deIiIXAQoDJkydHe3t7tzI3LV3O9evraW7vbb6w+/kHWqVSoVa7h4IWa9uXgc8BR6b1Y+nl7FlSefa8pnTM8j7Vs+fTalWiNwOjVtNfyb5Rg73BNrAcqMFUPZ+avwamSHo98FvgDGAdcD9wHsU9hJnA8lR+RVr/l7T9vogISSuAb0n6EvBmYALwACBggqTjKZLD+bw6WjNrOEkfAnZGxEOS2ptZl94MjFpNfyX7i+feXX9lemHOpH2DamA5UIOpPv8GImKtpLuAnwH7gJ9TdNq7gWWSvpBit6ZdbgVuT1PrXRQf+kTEBkl3Ukyp9wGXRcQfASRdDqyieHJqUURs6Gt9zfrBe4G/ljQdOBw4CriRJsyezRqlrjQZEfOAeVXhTbx6fbVc9hXgw5njXEtxjbc6vhJYWU8dzfpLRFwFXAWQZhKfjYgLJf0znj3bENWYuZTZ0HYlnj3bEOUkYdYHEVEBKmnZs2cbsvzdTWZmluUkYWZmWU4SZmaW5SRhZmZZThJmZpblJGFmZllOEmZmluUkYWZmWU4SZmaW5SRhZmZZThJmZpblJGFmZllOEmZmluUkYWZmWU4SZmaW5SRhZmZZThJmZpblJGFmZllOEmZmluUkYWZmWU4SZmaW5SRhZmZZThJmZpblJGFmZllOEmZmluUkYWZmWU4SZmaW5SRhZmZZThJmZpblJGFmZll1JQlJIyXdJemXkh6X9B5Jx0haLWlj+jkqlZWkBZI6JD0i6eTScWam8hslzSzFT5G0Pu2zQJLqqa+ZmR2YemcSNwI/jIh3Au8CHgfmAvdGxATg3rQOcA4wIb1mA7cASDoGmAecBpwKzOtKLKnMpaX9ptVZX7O6SDpc0gOSfiFpg6S/T/HjJa1NA5pvSzo0xQ9L6x1p+/jSsa5K8ScknV2KT0uxDklzq+tg1kh9ThKSjgY+ANwKEBG/j4g9wAxgcSq2GDg3Lc8AlkRhDTBS0mjgbGB1ROyKiN3AamBa2nZURKyJiACWlI5l1iy/A06PiHcBJ1H01SnAdcANEfF2YDcwK5WfBexO8RtSOSRNBM4HTqAY/HxV0jBJw4CbKQZVE4ELUlmzphhex77HA88C35D0LuAh4AqgLSK2pzLPAG1peQywpbT/1hTbX3xrjXg3kmZTzE5oa2ujUql0K9M2AuZM2tf71tWh1vkHWmdnZ1PO2wit1LY0YOlMq4ekVwCnAx9N8cXA1RQz4RlpGeAu4CvpsukMYFlE/A54SlIHxUwaoCMiNgFIWpbKPjZwrTLLqydJDAdOBj4VEWsl3cirl5aA4g0lKeqpYG9ExEJgIcDkyZOjvb29W5mbli7n+vX1NLf3Nl/Y/fwDrVKpUKvdQ0GrtS2N9h8C3k4x6n8S2BMRXaOQ8oDmz4OgiNgnaS9wbIqvKR22vE/1oOm0GnXocWDUavor2TdqsDfYBpYDNZiq51NzK7A1Itam9bsoksQOSaMjYnu6ZLQzbd8GjCvtPzbFtgHtVfFKio+tUd6sqSLij8BJkkYC3wPe2YQ69DgwajX9lewvnnt3/ZXphTmT9g2qgeVADab6fE8iIp4Btkh6RwqdQTElXgF0PaE0E1iellcAF6WnnKYAe9NlqVXAVEmj0g3rqcCqtO0FSVPS9Pyi0rHMmi7dg7sfeA/FPbauT5TygObPg6O0/WjgefY/aKoVN2uKep9u+hSwVNIjFDfxvgjMB86StBE4M60DrAQ2AR3A14BPAkTELuDzwIPpdU2Kkcp8Pe3zJHBPnfU1q4ukN6YZBJJGAGdRPNV3P3BeKlY9OOoaNJ0H3Jfua6wAzk9PPx1P8fTeAxTvgQnpaalDKW5urxj4lpnVVtdcKiIeBibX2HRGjbIBXJY5ziJgUY34OuDEeupo1s9GA4vTfYnXAXdGxA8kPQYsk/QF4Oekp/7Sz9vTjeldFB/6RMQGSXdSzL73AZely1hIupxihj0MWBQRGxrXPLPXaswFN7MhIiIeAd5dI76JV59OKsdfAT6cOda1wLU14ispZt5mTeev5TAzsywnCTMzy3KSMDOzLCcJMzPLcpIwM7MsJwkzM8tykjAzsywnCTMzy3KSMDOzLCcJMzPLcpIwM7MsJwkzM8tykjAzsywnCTMzy3KSMDOzLCcJMzPLcpIwM7MsJwkzM8tykjAzsywnCTMzy3KSMDOzLCcJMzPLcpIwM7MsJwkzM8tykjAzsywnCTMzy3KSMDOzLCcJMzPLcpIwM7MsJwkzM8uqO0lIGibp55J+kNaPl7RWUoekb0s6NMUPS+sdafv40jGuSvEnJJ1dik9LsQ5Jc+utq5mZHZj+mElcATxeWr8OuCEi3g7sBmal+Cxgd4rfkMohaSJwPnACMA34ako8w4CbgXOAicAFqaxZU0gaJ+l+SY9J2iDpihQ/RtJqSRvTz1EpLkkL0iDnEUknl441M5XfKGlmKX6KpPVpnwWS1PiWmr2qriQhaSzwQeDraV3A6cBdqchi4Ny0PCOtk7afkcrPAJZFxO8i4imgAzg1vToiYlNE/B5YlsqaNcs+YE5ETASmAJelgctc4N6ImADcm9ahGOBMSK/ZwC1QJBVgHnAaRT+f15VYUplLS/tNa0C7zLKG17n/l4HPAUem9WOBPRGxL61vBcak5THAFoCI2Cdpbyo/BlhTOmZ5ny1V8dNqVULSbIo3IW1tbVQqlW5l2kbAnEn7usUHQq3zD7TOzs6mnLcRWqVtEbEd2J6WX5T0OEVfnQG0p2KLgQpwZYoviYgA1kgaKWl0Krs6InYBSFoNTJNUAY6KiDUpvoRikHVPI9pnVkufk4SkDwE7I+IhSe39V6UDFxELgYUAkydPjvb27tW5aelyrl9fb07snc0Xdj//QKtUKtRq91DQim1L99TeDawF2lICAXgGaEvLfx4YJV0DoP3Ft9aI1zp/jwOjVtNfyb5Rg73BNrAcqMFUPZ+a7wX+WtJ04HDgKOBGYKSk4Wk2MRbYlspvA8YBWyUNB44Gni/Fu5T3ycXNmkbSG4DvAJ+OiBfKtw0iIiTFQNehNwOjVtNfyf7iuXfXX5lemDNp36AaWA7UYKrP9yQi4qqIGBsR4yluPN8XERcC9wPnpWIzgeVpeUVaJ22/L03DVwDnp6efjqe4DvsA8CAwIT0tdWg6x4q+1tesP0g6hCJBLI2I76bwjnQZifRzZ4rnBkD7i4+tETdrmoH4fxJXAp+R1EFxz+HWFL8VODbFP0O6uRcRG4A7gceAHwKXRcQf00zkcmAVxdNTd6ayZk2RHrS4FXg8Ir5U2lQeAFUPjC5KTzlNAfamy1KrgKmSRqUb1lOBVWnbC5KmpHNdVDqWWVP0y1wqIioUN+uIiE0UT2xUl3kF+HBm/2uBa2vEVwIr+6OOZv3gvcDHgPWSHk6x/wrMB+6UNAt4GvhI2rYSmE7xxN7LwCUAEbFL0ucpZssA13TdxAY+CdwGjKC4Ye2b1tZUjbngZjYERMRPgdz/WzijRvkALsscaxGwqEZ8HXBiHdU061f+Wg4zM8tykjAzsywnCTMzy3KSMDOzLCcJMzPLcpIwM7MsJwkzM8tykjAzsywnCTMzy3KSMDOzLCcJMzPLcpIwM7MsJwkzM8tykjAzsywnCTMzy3KSMDOzLCcJMzPLcpIwM7MsJwkzM8tykjAzsywnCTMzy3KSMDOzLCcJMzPLcpIwM7MsJwkzM8tykjAzsywnCTMzy3KSMDOzLCcJMzPLcpIwM7OsPicJSeMk3S/pMUkbJF2R4sdIWi1pY/o5KsUlaYGkDkmPSDq5dKyZqfxGSTNL8VMkrU/7LJCkehprZmYHpp6ZxD5gTkRMBKYAl0maCMwF7o2ICcC9aR3gHGBCes0GboEiqQDzgNOAU4F5XYkllbm0tN+0OuprVjdJiyTtlPRoKeaBkQ1ZfU4SEbE9In6Wll8EHgfGADOAxanYYuDctDwDWBKFNcBISaOBs4HVEbErInYDq4FpadtREbEmIgJYUjqWWbPcRvfBigdGNmT1yz0JSeOBdwNrgbaI2J42PQO0peUxwJbSbltTbH/xrTXiZk0TET8BdlWFPTCyIWt4vQeQ9AbgO8CnI+KF8uw4IkJS1HuOXtRhNsVIjba2NiqVSrcybSNgzqR9A10VgJrnH2idnZ1NOW8jDIK2NXxg1Js+32r669+xUe/jwfaZMVDvk7qShKRDKBLE0oj4bgrvkDQ6IrankdHOFN8GjCvtPjbFtgHtVfFKio+tUb6biFgILASYPHlytLe3dytz09LlXL++7pzYK5sv7H7+gVapVKjV7qFgMLWtUQOj3vT5VtNf/44Xz727/sr0wpxJ+wbVZ8ZAvU/qebpJwK3A4xHxpdKmFUDXjbiZwPJS/KJ0M28KsDeNvlYBUyWNStdlpwKr0rYXJE1J57qodCyzVrIjDYg4gIFRLt6rgZFZo9RzT+K9wMeA0yU9nF7TgfnAWZI2AmemdYCVwCagA/ga8EmAiNgFfB54ML2uSTFSma+nfZ4E7qmjvmYDxQMjG7L6PJeKiJ8CucfzzqhRPoDLMsdaBCyqEV8HnNjXOpr1N0l3UFwePU7SVoqnlOYDd0qaBTwNfCQVXwlMpxjkvAxcAsXASFLXwAi6D4xuA0ZQDIo8MLKmaswFN7MhIiIuyGzywMiGJH8th5mZZTlJmJlZlpOEmZllOUmYmVmWk4SZmWU5SZiZWZaThJmZZTlJmJlZlpOEmZllOUmYmVmWk4SZmWU5SZiZWZaThJmZZTlJmJlZlpOEmZllOUmYmVmWk4SZmWU5SZiZWZaThJmZZTlJmJlZlpOEmZllOUmYmVmWk4SZmWU5SZiZWZaThJmZZTlJmJlZlpOEmZllDW92Bcxs8Bs/9+4ey8yZtI+Le1HOWouThJlZk/UmyfakN0l48/wPHvBxfbnJzMyynCTMzCyr5ZOEpGmSnpDUIWlus+tjNtDc562VtHSSkDQMuBk4B5gIXCBpYnNrZTZw3Oet1bR0kgBOBToiYlNE/B5YBsxocp3MBpL7vLWUVn+6aQywpbS+FTitupCk2cDstNop6YkaxzoOeK7fa1iDrmvEWbppWPuaoNFte0sDz1WtP/t8S/nbQdZHh2J9e/hsqtnvWz1J9EpELAQW7q+MpHURMblBVWq4ody+ody2vupNn281g+3f0fUttPrlpm3AuNL62BQzG6rc562ltHqSeBCYIOl4SYcC5wMrmlwns4HkPm8tpaUvN0XEPkmXA6uAYcCiiNjQx8MNqql5Hwzl9g3ltr1GP/f5VjPY/h1dX0ARMRDHNTOzIaDVLzeZmVkTOUmYmVnWQZEkBuvXHEjaLGm9pIclrUuxYyStlrQx/RyV4pK0ILXxEUknl44zM5XfKGlmk9qySNJOSY+WYv3WFkmnpN9VR9pXjW2h7U+tvtxKDqR/toJMfa+WtC39jh+WNL1fThYRQ/pFcfPvSeCtwKHAL4CJza5XL+u+GTiuKvYPwNy0PBe4Li1PB+4BBEwB1qb4McCm9HNUWh7VhLZ8ADgZeHQg2gI8kMoq7XtOs//9/Np/X26l14H0z1Z4Zep7NfDZ/j7XwTCTGGpfczADWJyWFwPnluJLorAGGClpNHA2sDoidkXEbmA1MK3RlY6InwC7qsL90pa07aiIWBPFu2VJ6VhmPTrA/tl0mfoOiIMhSdT6moMxTarLgQrgR5IeSl/DANAWEdvT8jNAW1rOtbOV299fbRmTlqvj1jpq9eVWl+ufrezydIl2UX9dHjsYksRg9r6IOJniG0Evk/SB8sY0ah4SzzAPpbZYTfvty61ukPTPW4C3AScB24Hr++OgB0OSGLRfcxAR29LPncD3KC6d7UiXV0g/d6biuXa2cvv7qy3b0nJ13FpEpi+3ulz/bEkRsSMi/hgRfwK+Rj/9jg+GJDEov+ZA0hGSjuxaBqYCj1LUveupnpnA8rS8ArgoPRk0BdibpsqrgKmSRqXp59QUawX90pa07QVJU9JTTReVjmVNtp++3Opy/bMldSW05D/QX7/jZt+lb9CTANOBX1E85fTfml2fXtb5rRRPYv0C2NBVb+BY4F5gI/Bj4JgUF8Ufq3kSWA9MLh3r40BHel3SpPbcQTEF/gPFPYNZ/dkWYHJ6UzwJfIX0bQJ+Nf+V68ut9DqQ/tkKr0x9b0/vl0coEtzo/jiXv5bDzMyyDobLTWZm1kdOEmZmluUkYWZmWU4SZmaW5SRhZmZZThJmZpblJGFmZln/Hx6e33ca2f1bAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "lcrmT-4X0cHZ",
        "outputId": "0f6a71f6-985f-4ad2-9c8e-fa1d4b9ed19d"
      },
      "source": [
        "df.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text_count</th>\n",
              "      <th>summary_count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>102797.000000</td>\n",
              "      <td>102797.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>71.254531</td>\n",
              "      <td>9.440976</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>98.950307</td>\n",
              "      <td>1.413048</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>56.000000</td>\n",
              "      <td>9.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>58.000000</td>\n",
              "      <td>9.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>59.000000</td>\n",
              "      <td>10.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>11987.000000</td>\n",
              "      <td>16.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          text_count  summary_count\n",
              "count  102797.000000  102797.000000\n",
              "mean       71.254531       9.440976\n",
              "std        98.950307       1.413048\n",
              "min         1.000000       1.000000\n",
              "25%        56.000000       9.000000\n",
              "50%        58.000000       9.000000\n",
              "75%        59.000000      10.000000\n",
              "max     11987.000000      16.000000"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7_BeHsRWTILq"
      },
      "source": [
        "**Filter Outliers**\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UwUAr-2uMVMh",
        "outputId": "22d1bab7-0046-4434-e067-9e5af508221e"
      },
      "source": [
        "summary_records = 0\n",
        "for i in df.cleaned_summary:\n",
        "    if( (len(i.split()) >= 7) & (len(i.split()) <= 25) ):\n",
        "        summary_records += 1\n",
        "print('Summary data bewteen 7 to 16 words (%) = ',summary_records/df.shape[0] * 100)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Summary data bewteen 7 to 16 words (%) =  98.31901709193848\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PjNwZlPcN8It",
        "outputId": "5d157cc6-969e-48f0-e210-3540a91c1d34"
      },
      "source": [
        "text_records = 0\n",
        "for i in df.cleaned_text:\n",
        "    if( (len(i.split()) >= 5) & (len(i.split()) <= 150) ):\n",
        "        text_records += 1\n",
        "print('Text data bewteen 25 to 600 words (%) = ',text_records/df.shape[0] * 100)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Text data bewteen 25 to 600 words (%) =  96.01836629473623\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OuSokNeOQOg4"
      },
      "source": [
        "df_filtered = df[(df.summary_count <= 25) & (df.summary_count >= 7)]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "laQzoT1VRnJr"
      },
      "source": [
        "df_filtered = df_filtered[ (df_filtered.text_count >= 5)  & (df_filtered.text_count <= 100) ]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GROfpjGESpVS",
        "outputId": "24ec2aa7-ae3b-4eec-ca3b-c0f6f34c4ad9"
      },
      "source": [
        "df_filtered.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(96843, 6)"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v8TsoLSbTau5"
      },
      "source": [
        "df = df_filtered.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uawizbouTM95"
      },
      "source": [
        "**Adding Start and End Token to the summary**\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dxtSEAr0S-J7"
      },
      "source": [
        "df.cleaned_summary = df.cleaned_summary.apply(lambda x : 'sostok '+ x + ' eostok')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gMIIqhQ_TiLz"
      },
      "source": [
        "df = df[['cleaned_text','cleaned_summary']]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        },
        "id": "vM31IB4bTqec",
        "outputId": "cf4d5482-4cdf-4a2e-deae-e92ff00e2abc"
      },
      "source": [
        "df.head(2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>cleaned_text</th>\n",
              "      <th>cleaned_summary</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>saurav kant an alumnus of upgrad and iiit-b pg...</td>\n",
              "      <td>sostok upgrad learner switches to career in ml...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>kunal shah credit card bill payment platform c...</td>\n",
              "      <td>sostok delhi techie wins free food from swiggy...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                        cleaned_text                                    cleaned_summary\n",
              "0  saurav kant an alumnus of upgrad and iiit-b pg...  sostok upgrad learner switches to career in ml...\n",
              "1  kunal shah credit card bill payment platform c...  sostok delhi techie wins free food from swiggy..."
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S2F5WFKTnjbH"
      },
      "source": [
        "max_text_len = 100\n",
        "max_summary_len = 16"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "972Lg4mjTxcU"
      },
      "source": [
        "**SEQ2SEQ MODEL BUILDING**\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k7UN-DXvT0PB"
      },
      "source": [
        "x_train, x_test, y_train, y_test = train_test_split(df.cleaned_text, df.cleaned_summary, test_size=0.1, random_state=0, shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kUvugAYbhLRF"
      },
      "source": [
        "x_tokenizer = Tokenizer() \n",
        "x_tokenizer.fit_on_texts(list(x_train))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0uK6lGW_nVq_",
        "outputId": "439d9cf0-2b31-4d13-e25f-1cd10dea5580"
      },
      "source": [
        "thresh=1\n",
        "\n",
        "cnt=0\n",
        "tot_cnt=0\n",
        "freq=0\n",
        "tot_freq=0\n",
        "\n",
        "for key,value in x_tokenizer.word_counts.items():\n",
        "    tot_cnt=tot_cnt+1\n",
        "    tot_freq=tot_freq+value\n",
        "    if(value<thresh):\n",
        "        cnt=cnt+1\n",
        "        freq=freq+value\n",
        "    \n",
        "print(\"% of rare words in vocabulary:\",(cnt/tot_cnt)*100)\n",
        "print(\"Total Coverage of rare words:\",(freq/tot_freq)*100)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "% of rare words in vocabulary: 0.0\n",
            "Total Coverage of rare words: 0.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GX0X40JYnXij",
        "outputId": "50634264-00b2-4738-b045-08c8ad4fd14f"
      },
      "source": [
        "x_tokenizer = Tokenizer(num_words=tot_cnt-cnt) \n",
        "x_tokenizer.fit_on_texts(list(x_train))\n",
        "\n",
        "#convert text sequences into integer sequences (i.e one-hot encodeing all the words)\n",
        "x_train_seq = x_tokenizer.texts_to_sequences(x_train) \n",
        "x_test_seq = x_tokenizer.texts_to_sequences(x_test)\n",
        "\n",
        "#padding zero upto maximum length\n",
        "x_train_padded = pad_sequences(x_train_seq,  maxlen=max_text_len, padding='post')\n",
        "x_test_padded = pad_sequences(x_test_seq, maxlen=max_text_len, padding='post')\n",
        "\n",
        "#size of vocabulary ( +1 for padding token)\n",
        "x_voc = x_tokenizer.num_words + 1\n",
        "\n",
        "print(\"Size of vocabulary in X = {}\".format(x_voc))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Size of vocabulary in X = 78464\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZAhTClxfoBzR"
      },
      "source": [
        "y_tokenizer = Tokenizer()   \n",
        "y_tokenizer.fit_on_texts(list(y_train))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0KbnLhtmoFBJ",
        "outputId": "9a7f2592-4ad2-4cda-bf33-4bfcb8268aad"
      },
      "source": [
        "thresh=1\n",
        "\n",
        "cnt=0\n",
        "tot_cnt=0\n",
        "freq=0\n",
        "tot_freq=0\n",
        "\n",
        "for key,value in y_tokenizer.word_counts.items():\n",
        "    tot_cnt=tot_cnt+1\n",
        "    tot_freq=tot_freq+value\n",
        "    if(value<thresh):\n",
        "        cnt=cnt+1\n",
        "        freq=freq+value\n",
        "    \n",
        "print(\"% of rare words in vocabulary:\",(cnt/tot_cnt)*100)\n",
        "print(\"Total Coverage of rare words:\",(freq/tot_freq)*100)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "% of rare words in vocabulary: 0.0\n",
            "Total Coverage of rare words: 0.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rrU7zVQqoKmX",
        "outputId": "04b22fd3-25d6-4324-e2d9-f0e4173f1173"
      },
      "source": [
        "y_tokenizer = Tokenizer(num_words=tot_cnt-cnt) \n",
        "y_tokenizer.fit_on_texts(list(y_train))\n",
        "\n",
        "#convert text sequences into integer sequences (i.e one hot encode the text in Y)\n",
        "y_train_seq = y_tokenizer.texts_to_sequences(y_train) \n",
        "y_test_seq = y_tokenizer.texts_to_sequences(y_test) \n",
        "\n",
        "#padding zero upto maximum length\n",
        "y_train_padded = pad_sequences(y_train_seq, maxlen=max_summary_len, padding='post')\n",
        "y_test_padded = pad_sequences(y_test_seq, maxlen=max_summary_len, padding='post')\n",
        "\n",
        "#size of vocabulary\n",
        "y_voc = y_tokenizer.num_words +1\n",
        "print(\"Size of vocabulary in Y = {}\".format(y_voc))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Size of vocabulary in Y = 34118\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LtHIUVT3pZN3",
        "outputId": "a54868b6-88e0-41ca-87e8-f9870c094bd3"
      },
      "source": [
        "use_tpu = True #@param {type:\"boolean\"}\n",
        "\n",
        "if use_tpu:\n",
        "    assert 'COLAB_TPU_ADDR' in os.environ, 'Missing TPU; did you request a TPU in Notebook Settings?'\n",
        "\n",
        "if 'COLAB_TPU_ADDR' in os.environ:\n",
        "  TF_MASTER = 'grpc://{}'.format(os.environ['COLAB_TPU_ADDR'])\n",
        "else:\n",
        "  TF_MASTER=''\n",
        "\n",
        "tpu_address = TF_MASTER\n",
        "\n",
        "resolver = tf.distribute.cluster_resolver.TPUClusterResolver(TF_MASTER)\n",
        "tf.config.experimental_connect_to_cluster(resolver)\n",
        "tf.tpu.experimental.initialize_tpu_system(resolver)\n",
        "strategy = tf.distribute.experimental.TPUStrategy(resolver)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.93.102.10:8470\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.93.102.10:8470\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n",
            "WARNING:absl:`tf.distribute.experimental.TPUStrategy` is deprecated, please use  the non experimental symbol `tf.distribute.TPUStrategy` instead.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6spjrRT6tRoD"
      },
      "source": [
        "from keras import backend as K \n",
        "import gensim\n",
        "from numpy import *\n",
        "import numpy as np\n",
        "import pandas as pd \n",
        "import re\n",
        "from bs4 import BeautifulSoup\n",
        "from keras.preprocessing.text import Tokenizer \n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from nltk.corpus import stopwords\n",
        "from tensorflow.keras.layers import Input, LSTM, Embedding, Dense, Concatenate, TimeDistributed, Attention\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.callbacks import EarlyStopping\n",
        "import warnings\n",
        "pd.set_option(\"display.max_colwidth\", 200)\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "K.clear_session()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "81GFVKiEtCon",
        "outputId": "b54c2cd3-a8a3-4a9b-d0f4-3c539ed9bcce"
      },
      "source": [
        "with strategy.scope():\n",
        "\n",
        "  latent_dim = 300\n",
        "  embedding_dim = 200\n",
        "\n",
        "  # Encoder\n",
        "  encoder_inputs = Input(shape=(max_text_len,))\n",
        "\n",
        "  #embedding layer\n",
        "  enc_emb =  Embedding(x_voc, embedding_dim,trainable=True)(encoder_inputs)\n",
        "\n",
        "  #encoder lstm 1\n",
        "  encoder_lstm1 = LSTM(latent_dim,return_sequences=True,return_state=True,dropout=0.4,recurrent_dropout=0.4)\n",
        "  encoder_output1, state_h1, state_c1 = encoder_lstm1(enc_emb)\n",
        "\n",
        "  #encoder lstm 2\n",
        "  encoder_lstm2 = LSTM(latent_dim,return_sequences=True,return_state=True,dropout=0.4,recurrent_dropout=0.4)\n",
        "  encoder_output2, state_h2, state_c2 = encoder_lstm2(encoder_output1)\n",
        "\n",
        "  #encoder lstm 3\n",
        "  encoder_lstm3=LSTM(latent_dim, return_state=True, return_sequences=True,dropout=0.4,recurrent_dropout=0.4)\n",
        "  encoder_outputs, state_h, state_c= encoder_lstm3(encoder_output2)\n",
        "  \n",
        "\n",
        "  # Set up the decoder, using `encoder_states` as initial state.\n",
        "  decoder_inputs = Input(shape=(None,))\n",
        "\n",
        "  #embedding layer\n",
        "  dec_emb_layer = Embedding(y_voc, embedding_dim,trainable=True)\n",
        "  dec_emb = dec_emb_layer(decoder_inputs)\n",
        "\n",
        "  decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True,dropout=0.4,recurrent_dropout=0.2)\n",
        "  decoder_outputs,decoder_fwd_state, decoder_back_state = decoder_lstm(dec_emb,initial_state=[state_h, state_c])\n",
        "\n",
        "  #dense layer\n",
        "  decoder_dense =  TimeDistributed(Dense(y_voc, activation='softmax'))\n",
        "  decoder_outputs = decoder_dense(decoder_outputs)\n",
        "\n",
        "  # Define the model \n",
        "  model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
        "\n",
        "  model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 100)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding (Embedding)           (None, 100, 200)     15692800    input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lstm (LSTM)                     [(None, 100, 300), ( 601200      embedding[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "input_2 (InputLayer)            [(None, None)]       0                                            \n",
            "__________________________________________________________________________________________________\n",
            "lstm_1 (LSTM)                   [(None, 100, 300), ( 721200      lstm[0][0]                       \n",
            "__________________________________________________________________________________________________\n",
            "embedding_1 (Embedding)         (None, None, 200)    6823600     input_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lstm_2 (LSTM)                   [(None, 100, 300), ( 721200      lstm_1[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "lstm_3 (LSTM)                   [(None, None, 300),  601200      embedding_1[0][0]                \n",
            "                                                                 lstm_2[0][1]                     \n",
            "                                                                 lstm_2[0][2]                     \n",
            "__________________________________________________________________________________________________\n",
            "time_distributed (TimeDistribut (None, None, 34118)  10269518    lstm_3[0][0]                     \n",
            "==================================================================================================\n",
            "Total params: 35,430,718\n",
            "Trainable params: 35,430,718\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jVaM11cdtaYq"
      },
      "source": [
        "model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy', metrics='sparse_categorical_accuracy')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NQigbEFttfPB"
      },
      "source": [
        "es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z19d9ALqtgIi",
        "outputId": "db5d42f6-4814-463e-86a2-a45b08ca5fc6"
      },
      "source": [
        "history=model.fit( [x_train_padded,y_train_padded[:,:-1]], y_train_padded.reshape(y_train_padded.shape[0],y_train_padded.shape[1], 1)[:,1:] ,\n",
        "                  epochs=100, callbacks=[es], batch_size=128, \n",
        "                  validation_data=([x_test_padded,y_test_padded[:,:-1]], y_test_padded.reshape(y_test_padded.shape[0],y_test_padded.shape[1], 1)[:,1:]) )"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/100\n",
            "681/681 [==============================] - 143s 170ms/step - loss: 5.2624 - sparse_categorical_accuracy: 0.3662 - val_loss: 4.8939 - val_sparse_categorical_accuracy: 0.3907\n",
            "Epoch 2/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 4.8793 - sparse_categorical_accuracy: 0.3886 - val_loss: 4.6083 - val_sparse_categorical_accuracy: 0.4035\n",
            "Epoch 3/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 4.5835 - sparse_categorical_accuracy: 0.4038 - val_loss: 4.3522 - val_sparse_categorical_accuracy: 0.4179\n",
            "Epoch 4/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 4.3512 - sparse_categorical_accuracy: 0.4186 - val_loss: 4.1836 - val_sparse_categorical_accuracy: 0.4310\n",
            "Epoch 5/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 4.1800 - sparse_categorical_accuracy: 0.4309 - val_loss: 4.0596 - val_sparse_categorical_accuracy: 0.4395\n",
            "Epoch 6/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 4.0456 - sparse_categorical_accuracy: 0.4414 - val_loss: 3.9554 - val_sparse_categorical_accuracy: 0.4505\n",
            "Epoch 7/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.9346 - sparse_categorical_accuracy: 0.4516 - val_loss: 3.8827 - val_sparse_categorical_accuracy: 0.4570\n",
            "Epoch 8/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.8374 - sparse_categorical_accuracy: 0.4599 - val_loss: 3.8125 - val_sparse_categorical_accuracy: 0.4634\n",
            "Epoch 9/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.7497 - sparse_categorical_accuracy: 0.4680 - val_loss: 3.7576 - val_sparse_categorical_accuracy: 0.4690\n",
            "Epoch 10/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.6734 - sparse_categorical_accuracy: 0.4751 - val_loss: 3.7119 - val_sparse_categorical_accuracy: 0.4724\n",
            "Epoch 11/100\n",
            "681/681 [==============================] - 89s 131ms/step - loss: 3.6063 - sparse_categorical_accuracy: 0.4817 - val_loss: 3.6805 - val_sparse_categorical_accuracy: 0.4758\n",
            "Epoch 12/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.5478 - sparse_categorical_accuracy: 0.4875 - val_loss: 3.6542 - val_sparse_categorical_accuracy: 0.4792\n",
            "Epoch 13/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.4992 - sparse_categorical_accuracy: 0.4938 - val_loss: 3.6265 - val_sparse_categorical_accuracy: 0.4831\n",
            "Epoch 14/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.4532 - sparse_categorical_accuracy: 0.4984 - val_loss: 3.6012 - val_sparse_categorical_accuracy: 0.4859\n",
            "Epoch 15/100\n",
            "681/681 [==============================] - 89s 131ms/step - loss: 3.4074 - sparse_categorical_accuracy: 0.5033 - val_loss: 3.5816 - val_sparse_categorical_accuracy: 0.4875\n",
            "Epoch 16/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.3658 - sparse_categorical_accuracy: 0.5077 - val_loss: 3.5675 - val_sparse_categorical_accuracy: 0.4897\n",
            "Epoch 17/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.3322 - sparse_categorical_accuracy: 0.5122 - val_loss: 3.5540 - val_sparse_categorical_accuracy: 0.4911\n",
            "Epoch 18/100\n",
            "681/681 [==============================] - 89s 131ms/step - loss: 3.2976 - sparse_categorical_accuracy: 0.5160 - val_loss: 3.5460 - val_sparse_categorical_accuracy: 0.4926\n",
            "Epoch 19/100\n",
            "681/681 [==============================] - 89s 131ms/step - loss: 3.2662 - sparse_categorical_accuracy: 0.5199 - val_loss: 3.5280 - val_sparse_categorical_accuracy: 0.4954\n",
            "Epoch 20/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.2319 - sparse_categorical_accuracy: 0.5240 - val_loss: 3.5252 - val_sparse_categorical_accuracy: 0.4963\n",
            "Epoch 21/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.2039 - sparse_categorical_accuracy: 0.5276 - val_loss: 3.5205 - val_sparse_categorical_accuracy: 0.4962\n",
            "Epoch 22/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.1772 - sparse_categorical_accuracy: 0.5310 - val_loss: 3.5134 - val_sparse_categorical_accuracy: 0.4974\n",
            "Epoch 23/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.1534 - sparse_categorical_accuracy: 0.5342 - val_loss: 3.5119 - val_sparse_categorical_accuracy: 0.4982\n",
            "Epoch 24/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.1269 - sparse_categorical_accuracy: 0.5370 - val_loss: 3.4993 - val_sparse_categorical_accuracy: 0.4990\n",
            "Epoch 25/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.0954 - sparse_categorical_accuracy: 0.5399 - val_loss: 3.4876 - val_sparse_categorical_accuracy: 0.4992\n",
            "Epoch 26/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.0696 - sparse_categorical_accuracy: 0.5421 - val_loss: 3.4878 - val_sparse_categorical_accuracy: 0.4984\n",
            "Epoch 27/100\n",
            "681/681 [==============================] - 89s 130ms/step - loss: 3.0462 - sparse_categorical_accuracy: 0.5453 - val_loss: 3.4935 - val_sparse_categorical_accuracy: 0.4991\n",
            "Epoch 00027: early stopping\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "V4xSJKBuRuz5",
        "outputId": "92108ea8-2914-46a0-b1ba-b117c96e1e21"
      },
      "source": [
        "plt.plot(history.history['loss'], label='train')\n",
        "plt.plot(history.history['val_loss'], label='test')\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU1f3/8deZyb6vZAUSwg6BsEUWsSwqOy5Y3LBitWhrW9t+a9XW6rc++lX7a+tWl7rRqlitglY2LdKKArIvgbAICQTIRvZ9z5zfH3cSYiCTkGUmM/k8H495zM29Zyaf26nvXM6cc67SWiOEEML5mRxdgBBCiO4hgS6EEC5CAl0IIVyEBLoQQrgICXQhhHARbo76xWFhYTouLs5Rv14IIZzSvn37CrTW4Zc65rBAj4uLY+/evY769UII4ZSUUmfaOiZdLkII4SIk0IUQwkVIoAshhItwWB+6EEJ0Rn19PZmZmdTU1Di6lB7l5eVFbGws7u7uHX6NBLoQwqlkZmbi7+9PXFwcSilHl9MjtNYUFhaSmZlJfHx8h18nXS5CCKdSU1NDaGioy4Y5gFKK0NDQy/5XiAS6EMLpuHKYN+nMOTpdoB84W8wfPjvu6DKEEKLXcbpAT80q5ZUt6XyTW+7oUoQQfVBJSQkvv/zyZb9u/vz5lJSU9EBFFzhdoM8dHYVJwfpD2Y4uRQjRB7UV6A0NDTZft3HjRoKCgnqqLMAJAz3c35OpCWGsS8lG7rYkhLC3hx9+mPT0dJKSkpg0aRLTp09n8eLFjBw5EoDrr7+eCRMmMGrUKF577bXm18XFxVFQUEBGRgYjRozgBz/4AaNGjeLaa6+lurq6W2pzymGLi8ZG8dCaw6RmlZEYG+jocoQQDvK7dUc4ml3Wre85MjqAxxeNavP4008/TWpqKgcPHmTLli0sWLCA1NTU5uGFK1euJCQkhOrqaiZNmsSSJUsIDQ391nucPHmS9957j9dff52lS5eyZs0ali1b1uXane4KHWDOqEjczUq6XYQQDpecnPytseIvvPACY8eOZfLkyZw7d46TJ09e9Jr4+HiSkpIAmDBhAhkZGd1Si1NeoQf5eDB9SDjrD+Xw0NzhmEyuP4RJCHExW1fS9uLr69u8vWXLFjZv3syOHTvw8fFhxowZlxxL7unp2bxtNpu7rcvFKa/Qweh2ySqp5sC5YkeXIoToQ/z9/Skvv/Qou9LSUoKDg/Hx8eH48ePs3LnTrrU55RU6wNUjIvB0M7EuJYcJA0McXY4Qoo8IDQ1l2rRpjB49Gm9vbyIiIpqPzZ07l7/+9a+MGDGCYcOGMXnyZLvWphw1UmTixIm6qze4uO+dfew7W8zOR2Zjlm4XIfqEY8eOMWLECEeXYReXOlel1D6t9cRLtXfaLheARWOjyS+vZdfpQkeXIoQQDufUgT5reD98PMysS8lxdClCCOFwTh3o3h5mrh4RwWepOdQ3WhxdjhBCOJRTBzoY3S7FVfVsTytwdClCCOFQTh/oVw0Nw9/LTbpdhBB9ntMHuqebmbmjItl0JJea+kZHlyOEEA7j9IEOsHBsNOW1DXx1It/RpQghXFxnl88FeO6556iqqurmii5wiUCfmhBKiK8H6w5Jt4sQomf15kB32pmiLbmbTcwbHclH+7OoqmvAx8MlTksI0Qu1XD73mmuuoV+/fnzwwQfU1tZyww038Lvf/Y7KykqWLl1KZmYmjY2N/Pa3v+X8+fNkZ2czc+ZMwsLC+OKLL7q9NpdJvoVjonl311n+cyyPRWOjHV2OEMIePn0Ycg9373tGJsK8p9s83HL53E2bNrF69Wp2796N1prFixfz1VdfkZ+fT3R0NBs2bACMNV4CAwN55pln+OKLLwgLC+vemq1cossFIDk+hH7+nrKkrhDCbjZt2sSmTZsYN24c48eP5/jx45w8eZLExEQ+//xzHnroIbZu3UpgoH3u29ChK3SlVAZQDjQCDa3XEVDG7amfB+YDVcByrfX+7i3VNrNJsWBMFO/uOktZTT0BXu72/PVCCEewcSVtD1prHnnkEe69996Lju3fv5+NGzfy6KOPMnv2bB577LEer+dyrtBnaq2T2lgUZh4wxPpYAbzSHcVdroVjoqlrsPD5kfOO+PVCiD6g5fK5c+bMYeXKlVRUVACQlZVFXl4e2dnZ+Pj4sGzZMh588EH2799/0Wt7Qnf1oV8HvK2NpRt3KqWClFJRWmu7DjsZPyCImCBv1h/KZsmEWHv+aiFEH9Fy+dx58+Zx2223MWXKFAD8/PxYtWoVaWlpPPjgg5hMJtzd3XnlFeMad8WKFcydO5fo6Oge+VK0Q8vnKqVOA8WABl7VWr/W6vh64Gmt9Tbrz/8BHtJat7k+bncsn3spT316jDe3nmbPb64m2Nej299fCOFYsnxu15fPvVJrPR6ja+V+pdRVnSlOKbVCKbVXKbU3P79nJgEtGhNNg0Xz2ZHcHnl/IYTorToU6FrrLOtzHvAxkNyqSRbQv8XPsdZ9rd/nNa31RK31xPDw8M5V3I5R0QHEh/myLkVGuwgh+pZ2A10p5auU8m/aBq4FUls1Wwt8TxkmA6X27j9vopRi0Zgodp4qJK/84puzCiGcn6PutGZPnTnHjlyhRwDblFIpwG5gg9b6M6XUfUqp+6xtNgKngDTgdeBHl11JN1o0NhqLhk8PS7eLEK7Gy8uLwsJClw51rTWFhYV4eXld1uvaHeWitT4FjL3E/r+22NbA/Zf1m3vQkAh/hkX4sy4lmzunxjm6HCFEN4qNjSUzM5Oe+h6ut/Dy8iI29vJG67nM1P/WFo2N4k+bTpBdUk10kLejyxFCdBN3d3fi4+MdXUav5DJT/1tbOMZYz2WDrMAohOgjXDbQ48J8SYwJZJ2s7SKE6COcL9DTNsPq70MHvhBZNDaKQ5mlZBRU2qEwIYRwLOcL9MpCSF0DJz9vt+mCpm6Xw9LtIoRwfc4X6KNvhIBY2P58u01jgryZFBfMh3vPYbG47hAnIYQAZwx0sztM+RGc2QaZ7a8Fc8eUODIKq/jv8Tw7FCeEEI7jfIEOMP574BXYoav0eaMjiQ704o1tp+xQmBBCOI5zBrqnP0y8G46tg8J0m03dzSbunBrHzlNFpGaV2qlAIYSwP+cMdIAr7jO6X3a82G7TW5IH4ONhZuW203YoTAghHMN5A90/AsbeCgfehQrbU4ADvd1ZOrE/6w5lc75MFuwSQrgm5w10gKk/gcY62P1au03vmhZHg0Xz9o6MHi9LCCEcwbkDPWwIDF9gBHpthc2mA0N9uXZkBO/uOkt1XaOdChRCCPtx7kAHmPYA1JTAgVXtNr37ykGUVNWzZn+mHQoTQgj7cv5A758M/SfDjpegscFm00lxwYyJDWTlttMy0UgI4XKcP9DBuEovPQtH/2WzmVKKu6+M51RBJVtOyEQjIYRrcY1AHzoXwobC9ufaXbRrfmIUkQFevLFVhjAKIVyLawS6yWSMeMk9DKe+sNnU3Wxi+bQ4vk4v5Gh2mZ0KFEKInucagQ4w5mbwi4DtL7Tb9NZJA/B2N/OmTDQSQrgQ1wl0N0+Y/EPjCj0nxWbTQB93lk6MZW1KFnky0UgI4SJcJ9ABJtwFHn4dukq/a1o8DRbNOzvP2KEwIYToea4V6N5BMGE5HPkYim0HdVyYL1ePiGDVzjPU1MtEIyGE83OtQAeY/CNQCna+3G7Te66Mp7iqno/2Z9mhMCGE6FmuF+iBMZC4FPa/DVVFNpsmx4cwOiaAN7edkolGQgin53qBDsYQxvoq2POGzWZKKe65chDp+ZV8edL2io1CCNHbuWagR4yEIdfCrlehvtpm0/mJUUQEePKmTDQSQjg51wx0MJYDqCqAg/+w2czDzbij0ba0Ao7nykQjIYTzct1AHzgNosfD138Bi+1RLLclWycayVW6EMKJuW6gK2VcpRefhuPrbTYN8vHgpgmxfHIwm/zyWjsVKIQQ3ct1Ax1gxCIIjoetf2530a67psVRb7HIRCMhhNNy7UA3meGqXxpLARzfYLPpoHA/Zg/vJxONhBBOy7UDHWDMLRCSAF88CRaLzaZ3XzmIoso6PtwndzQSQjgf1w90sxvMeBjyjrR7A4zJg0JIjgvh+c0nKK+pt1OBQgjRPVw/0AFGL4Hw4bDlaZsjXpRSPLpwBAUVdby8Jd2OBQohRNf1jUA3mWHGI1DwDRxebbPpmNggbhwXw5vbTnOuqMpOBQohRNf1jUAHGLEYIhJhy1PQaLs75cG5wzAp+MNnx+1UnBBCdF2HA10pZVZKHVBKXTSoWym1XCmVr5Q6aH3c071ldgOTCWb+2hiXnvKezaZRgd7ce1UC6w/lsO+M7QW+hBCit7icK/QHgGM2jv9Ta51kfdheFctRhs0zZo9++UdoqLPZ9N7vDCIiwJMn1h+TlRiFEE6hQ4GulIoFFgC9M6g7SimY9RsoPQsH3rbZ1MfDjQfnDCflXAnrDmXbqUAhhOi8jl6hPwf8CrA1kHuJUuqQUmq1Uqr/pRoopVYopfYqpfbm5ztoudqE2dB/Mnz1Z6i3fT/RG8fFMDomgD98epzqOplsJITo3doNdKXUQiBPa73PRrN1QJzWegzwOfDWpRpprV/TWk/UWk8MDw/vVMFd1nSVXp4N+/5ms6nJpPjtgpFkl9bw5rZTdipQCCE6pyNX6NOAxUqpDOB9YJZSalXLBlrrQq1106pWbwATurXK7hZ/FcRNN9Z4qau02fSKQaHMHRXJy1vSySuzfUUvhBCO1G6ga60f0VrHaq3jgFuA/2qtl7Vso5SKavHjYmx/edo7zHoUKvNh9+vtNn143nDqGy38edMJOxQmhBCd0+lx6EqpJ5RSi60//lQpdUQplQL8FFjeHcX1qAGTYfDVsP15qC232TQuzJflU+P4YN85jmSX2qlAIYS4PJcV6FrrLVrrhdbtx7TWa63bj2itR2mtx2qtZ2qtnWNGzsxfQ3UR7Pxru01/PGsIQd7u/H79MXQ7S/EKIYQj9J2ZopcSMwGGzYcdf4HqEptNA73d+fk1Q9lxqpDNx/LsVKAQQnRc3w50MK7Sa0phx0vtNr0teQCD+/nx5MZj1DXYXopXCCHsTQI9MhFGXg87X4bKQptN3cwmfjN/BKcLKuXORkKIXkcCHYyVGOsq4evn2286LJzpQ8J44T8nKamyvXyAEELYkwQ6QL/hkPhdYwhjhe3+caUUjy4YSXlNPc9tPmmnAoUQon0S6E1mPAwNtbDt2XabDov055bkAazaeYb0/Ao7FCeEEO2TQG8SmgBJt8KeN6Gs/cW4fnHNULzczTy1sffPoRJC9A0S6C1d9StAw7oH2r2hdJifJ/fPHMzmY3msTZHVGIUQjieB3lLwQJjzJJzcBFv/1G7ze6bHM2FgMA+vOURanu3ZpkII0dMk0FubdA+MuRm+eBLSNtts6m428dJt4/F2N/PDVfuprG2wU5FCCHExCfTWlIKFz0G/kbDmHii2Pd48MtCLF24dR1p+Bb/5+LAsCyCEcBgJ9Evx8IGb3wFLI3zwvXZvhDFtcBi/uHoo/zqYzbu7ztqpSCGE+DYJ9LaEJsANf4Wcg/Dpr9ptfv/MwcwYFs4T645yKNP2ujBCCNETJNBtGb4ArvwF7H8L9r9js6nJpHh2aRLh/p78cNV+mUUqhLA7CfT2zHoU4r8DG/4Hsg/abBrs68FLt48nr7yGX3yQgsUi/elCCPuRQG+PyQw3rQTfMPjgDqgqstk8qX8Qv104kv8ez+OVL9PtVKQQQkigd4xvGCx9G8py4ON72510dMfkgSweG82fN33D1+kFdipSCNHXSaB3VOxEmPe0Menoqz/abKqU4qkbExkU7sdP3zvAebm5tBDCDiTQL8fEu2HMLbDlKThpe9KRr6cbr9w+nqq6Rn78j/3UN8oNMYQQPUsC/XIoBQufhYhRsObudicdDYnw56kbE9mTUcwf//2NnYoUQvRVEuiXq2nSkdYdmnR0XVIMd0weyGtfneKz1Fw7FSmE6Isk0DsjZBDc+Kox6WjjL41wt+HRhSMYGxvIgx+mcKaw0k5FCiH6Ggn0zho2D6b/Eg68A5sftxnqnm5mXrp9PCaTYsXb+yiulElHQojuJ4HeFTN/Y3xRuv152PALm8MZY4N9ePn28ZwurOSOlbsora63Y6FCiL5AAr0rTCZY8GeY9jPYuxL+dR80tr2E7rTBYby6bALf5JbzvZW7Ka+RUBdCdB8J9K5SCq75Hcx+DA79Ez6807g3aRtmDu/HS7eN50hWKcv/tkfWUBdCdBsJ9O4y/X9g3h/h+Hr4x81Q1/aXn9eOiuQvt47j4LkS7vr7HqrqJNSFEF0ngd6drlgB178Cp7+Ed26A6raX0Z2XGMUzS8eyN6OIH7y9l5r6RjsWKoRwRRLo3S3pNvju3yFrP7y1CCrbXsvluqQY/njTWL5OL+Ted/ZR2yChLoToPAn0njDyOrj1fSg4AX+bB2XZbTZdMiGWp29M5MsT+dz/7n7qGmSJACFE50ig95QhV8Oyj4wVGlfOhaLTbTa9edIAfn/9aDYfy+Mn78m6L0KIzpFA70lx0+DOtVBbZoR63vE2my6bPJDHF43k30fO8/N/HqRBQl0IcZkk0HtazHhYvhHQRvdL9oE2m941LZ7fzB/B+kM5PLj6EI1yxyMhxGWQQLeHiJFw16fg4Qd/WwBHPm6z6Q+uGsSDc4bx8YEsHl5zSG5jJ4ToMAl0ewlNgHs+h8jR8OFy+PyxNmeV3j9zMA/MHsKH+zL5nw9TZPSLEKJDJNDtyT8S7lx/Yf2Xd5dAZeElm/7s6iH88tqhfHwgi2Vv7KJIFvQSQrSjw4GulDIrpQ4opdZf4pinUuqfSqk0pdQupVRcdxbpUtw8YOEzsPhFOLMDXpsBOSkXNVNK8eNZQ/jLreM4lFnK9S9tJy2v3P71CiGcxuVcoT8AHGvj2N1AsdZ6MPAs8IeuFubyxt8B3/8UdCO8eS2kvH/JZovGRvP+islU1TVww8tfs/Vkvp0LFUI4iw4FulIqFlgAvNFGk+uAt6zbq4HZSinV9fJcXMwEWPElxE6Cj++Fjb+CxotXYBw3IJh/3T+NmCBvlv9tD6t22r71nRCib+roFfpzwK+AtgZHxwDnALTWDUApENq6kVJqhVJqr1Jqb36+XGkC4BcOd/wLJt8Pu1+FtxZDRd5FzWKDffjwvilcNSSMR/+VyhPrjsqwRiHEt7Qb6EqphUCe1npfV3+Z1vo1rfVErfXE8PDwrr6d6zC7wdwn4cY3jHHqr14F5/Zc1Mzfy5037pzE96fFs3L7aX7w9l4qZPldIYRVR67QpwGLlVIZwPvALKXUqlZtsoD+AEopNyAQuPTwDdG2Md81hjaaPeDv82Hf3y9qYjYpHls0kt9fP5ovT+Rz0ytfk1VSbf9ahRC9TruBrrV+RGsdq7WOA24B/qu1Xtaq2VrgTuv2TdY20h/QGZGJsGILxF0J6x6Af94BRacuarZs8kD+ftckskqque7F7Rw4W2z3UoUQvUunx6ErpZ5QSi22/vgmEKqUSgN+ATzcHcX1WT4hcPtqmPUopG2GF5Phs19DVdG3mk0fEs5HP5yKt4eJW17byfpDba/qKIRwfcpRF9ITJ07Ue/fudcjvdiplObDlSTiwCjz94aoHIXkFuHk2NymsqOW+VfvYk1HMvVcN4pdzhuFuljljQrgipdQ+rfXESx2T/+p7u4AoWPwXuG+bMbxx06Pw4iRIXQPWP8ahfp6suucKbr9iAK9+dYqlr+7gXFGVgwsXQtibBLqziBgFy9bAHR8bV+qrvw9vXG3MNgU83cz83w2JvHTbeNLOV7Dgha18lprr4KKFEPYkge5sEmbBvV/BdS9DWRb8bS78cxkUpgOwYEwUG346nfgwX+5btY/HP0mV+5UK0UdIoDsjkxnG3Q4/2Q8zH4X0L+ClZGOmaWUBA0J9+PC+qdxzZTxv7TjDkle+5nRBpaOrFkL0MPlS1BVU5MGWp4xx6+4+MPUnMOV+8PRn89Hz/HJ1CvUNFp68MZHrkmIcXa0QogvkS1FX59cPFj4LP9oFCTONcH8+CXa9ytVDg9n40+mMjA7ggfcP8tDqQ1TXSReMEK5IAt2VhA+Fm1fBPf+BfiPg01/BixOJPruO9+5J5sczB/PBvnMsfnEbJ87LUrxCuBoJdFcUOxHuXAe3rwGvAPjoB7i9PoNfDjrL23dNoriqjsUvbuO93WeRCb1CuA4JdFelFAy5GlZ8BUvehLpyePcmpm+/i03f9WbiwBAe+egw3//7HnJLaxxdrRCiG0iguzqTCRJvgvv3wPw/QcE3hLy3gHd8n+eZmZ7sOFXINc9+yep9mXK1LoSTk1EufU1tBex8Gba/AHXlVPf/Di+Uz+DV3CHMGB7JkzckEhno5egqhRBtsDXKRQK9r6oshL1vwt6/QXk25V5RvFY1g7Wm2fxk0RSWjI9BbjolRO8jgS7a1lgP32yE3a9DxlbqcGdd4xUcjb2FFbd+l4gAuVoXojeRQBcdk3cMvfsN6g/8A4/GKlJJoCbpLibMvxvl4ePo6oQQSKCLy1VTRsHXb1O9/a/0bzxHhckfNf57+E65G0ITHF2dEH2aBLrolMZGC//esBrzvteZrfbihgUdPQ41+iYYfSMERDu6RCH6HAl00SWnCyp5+p//oX/2Z9zitZPBjemAMm6TN3oJjLzOuMuSEKLHSaCLLrNYNJ+kZPHUxuP4VmTwUEwqVzduxa04DUxukDAbEr8Lw+aBp5+jyxXCZdkKdDd7FyOck8mkuGFcLNeOjOSlL9L46dYY3Mxz+N9JDdzovhO3ox/DyX+Dm7cR6ok3Qfx3JNyFsCO5Qhedcqawkt9vOMbnR88zIMSHR+cP4xq/06jUNXDkY6guAmWGqLEwcKrxGDBFumaE6CLpchE9ZuvJfH637ihpeRVMHxLGYwtHMiTMCzK2QsY24xZ5WfugsdZ4Qfhwa7hPhYFTIDDWsScghJORQBc9qr7Rwjs7zvDs5hNU1TXyvSkD+dnVQwn0drc2qIHsA3BmO5zdAWd3GYuFAQQNMMJ9wGSIGQ/hI8DNw3EnI0QvJ4Eu7KKwopY/bTrB+3vOEuzjwf0zB3P7FQPwcjd/u6GlEXIPG+F+5mvjuTLfOGb2MG6IHT0OopIgOklCXogWJNCFXaVmlfJ/G46x41QhEQGe/HjmYJZO6o+nm/nSL9Aaik5BzkHIPmh9ToHaUuO42QMiRhvh3hTy/UaC2d1+JyVELyGBLhzi6/QCntl0gr1niokJ8uYnswazZEIs7uYOrNpssUDxaWu4H7AG/aELIe/mBZFjIGaC9TEeQgYZ68AL4cIk0IXDaK356mQBz2z6hpTMUgaE+PDA7CFclxSNW0eCvaWmkM8+YDyy9huBX19lHPcKahHw1pD369f9JyWEA0mgC4fTWvOfY3k88/kJjuaUMSjclwdmD2HRmGhMpi5cVTc2QP4xYyRN1j4j5POOgrYYxwMHGMEenWRc0UeNBd+w7jkpIRxAAl30GhaLZtPRXJ79/CTfnC9naIQfP796KHNGRXYt2Fuqq4SclBYhvw9Kzl44HhBjDXdrwEeOMYZPSneNcAIS6KLXsVg06w/n8NzmE5zKr2REVAD3XjWIBWOiOtbHfrmqioyRNTkpkHvI6I8vOAFY///vHWIEfKT1ET4UQhJkpqvodSTQRa/V0GhhbUo2L36Rxqn8SiIDvFg+LY5bJw0g0KeHR7HUVcL5I0bINwV93jForLvQxj8KQgdf/AgeKKNshENIoItez2LRfHkinze2nWJ7WiE+HmaWTuzPXdPiGBjqa79CGuqg8CQUnITCNChMtz6nGcsZNDG5QXCcNdzjjaWEWz78o8DN0351iz5DAl04laPZZbyx7RTrUrJpsGiuHRnBPdMHMXFgsGPvc1pVdCHcmx/pUHQa6isvbu8TBgFRRp+9v/U5IMq6HQ3+kcbIHOm7F5dBAl04pfNlNby9I4N3d52lpKqesbGB3D19EPNGR/ZMP3tnaQ21ZVCWA2VZUJ4DZdnGdpl1uzwbqgovfq2blxHs/taA94+6EPr+keAXaSxo5hkAZlkcVUigCydXVdfAmv1ZrNx2mtMFlUQHerFsykBuHBdLZKAT3cS6vsYI9vJcI/TLc61hn9tiX86FcfWtufuCVwB4BRoPz6btFvu8Q4yuoJB4418EpjZm5wqnJYEuXILFovnv8Tze2HaKnaeKMCmYNjiMmyYY67R7e7hAeDVd7TeFfcV5qC4x9tWUQk0J1DRtl7bYXwqWhm+/l9nDWPwsON6YRRsSb92Oh6CB4O5EfwxFsy4FulLKC/gK8MS4IcZqrfXjrdosB/4IZFl3vai1fsPW+0qgi67IKKjko/2ZrNmfRVZJNX6ebsxPjGTJ+FiS40Mc29fuCFpDfbWxyFnxaaNf/1vPGRdWuARAGf34vuHGaB2Tu/Fsdjf+EJjcWm17GD+7eYK7j9FV5O5tffYx/ji4eRv7Wm57BoCnvyyu1o26GugK8NVaVyil3IFtwANa650t2iwHJmqtf9zRoiTQRXewWDS7ThexZn8mnx7OobKukf4h3tw4LpYl42MZEOrj6BJ7B62NPvyi08ZCaE1hX10MlnpotD4s9cawzcYG47nlscZ6aKi5sLb95TB7GmP6Pf2tj4AW2/7g4Wf8YTCZQZmsz+YLPyszmEwt9pnBOwj8IqyPfn1mVFG3dbkopXwwAv2HWutdLfYvRwJdOFhVXQOfpeby0f4stqcXoDUkx4Vw4/gY5o+JIsBLxo13C0ujEez1NUZ/f0ON8a+D+mpoqL6wv74a6iqMbqHacqitsD43Paz766z7G2q6Vpd3cIuAjwD/CONL5aZt33Dw7We0M/WiL9UvU5cDXSllBvYBg4GXtNYPtTq+HHgKyAdOAD/XWp+7xPusAFYADBgwYMKZM2cu70yE6KDskmo+PpDFmv2ZnMqvxMPNxKxh/bguKZqZw/tdvEa7cDytjT8WurHVs8V4tNxnaTC+Tyg/b3zP0PQoz4WKPKjINY5d6l8TymwN93Dws4a8b5hxle/bz9jnFQQ0ddvpb9d44YcL+xrrjD9idZXW5ypjKGtdVav91rqP91UAAAzJSURBVOfke2HY3E79z9SdV+hBwMfAT7TWqS32hwIVWutapdS9wM1a61m23kuu0IU9aK1JySzlk4NZrEvJoaCiFj9PN+aMiuS6pGimJoRe/qqPwjlobXxZ3BTwlflQkQ+Veca+yvxv7+vqvxDa4u4LHj5Gl5KHr/E87QEYubhTb9eto1yUUo8BVVrrP7Vx3AwUaa0Dbb2PBLqwt0aLZkd6IWtTsvg0NZfymgbC/DxYkBjF4qQYxg8I6ntfpgqD1ka3T1PI15Ry4QqdVpO/1MWbZo9LB7e7d7dPHOvql6LhQL3WukQp5Q1sAv6gtV7fok2U1jrHun0D8JDWerKt95VAF45UU9/Ilm/yWZuSxeZjedQ1WIgN9mbx2GgWJ0UzPDLA0SUKcUldDfQxwFuAGTABH2itn1BKPQHs1VqvVUo9BSwGGoAijC9Nj9t6Xwl00VuU19Sz6ch5PknJZntaAY0WzbAIfxYnRbN4bDT9Q2SkjOg9ZGKREB1UUFHLhkM5rE3JZt+ZYgDGDQhi8dhoFoyJop+/TMYRjiWBLkQnZBZXsS7FCPdjOWWYFExNCGPx2GjmjI4k0FuGQQr7k0AXootOni9nbUo2a1OyOVNYhYfZxIxh4SxOimb28AjXWHZAOAUJdCG6SdMwyLUHs1l/KJu88lp8PMx8Z2g4c0ZFMnN4P7lyFz1KAl2IHtBo0ew6XcjGwzlsOnKevPJa3M2KyYNCmTMqkmtHRtAvQPrcRfeSQBeih1ksmgPnSth0JJd/H8klo7AKpWD8gGDmjIpgzqhI+955SbgsCXQh7EhrzYnzFXyWaoT70ZwyAIZH+nPtqEhmDe/H6OgAmaEqOkUCXQgHOldUxb+P5LLpyHn2nClCa/D3dCM5PoQpCaFMSQhlRGQAJpPMUhXtk0AXopcoqKhlR3ohX6cXsiO9gIxC4+5EQT7uTI4PZergUKYMCmVwPz9ZhkBckgS6EL1Udkk1O9IL2XGqkB3phWSVVAMQ5ufJlIRQpiaEMi0hTNZ1F80k0IVwAlprzhVV83V6QXPA55Uby7/2D/FmWkIY0waHMTUhlFC/vnEzB3ExCXQhnJDWmvT8CranFbI9zQj58hrjvqEjogK4cnAoUweHkRwXgq+nm4OrFfYigS6EC2hotJCaXcb2tAK2pxWwN6OYukYL7mbFuP7BTBscxrTBoSTGBuLpJjNXXZUEuhAuqLqukb1nipqv4FOzS9EaPN1MJPUPIjk+hElxIYwfGIyfXMG7DFuBLp+yEE7K28PM9CHhTB8SDkBJVR07TxWxJ8N4vLwlnUZLGmaTYmRUAJPiQqwhHyx98C5KrtCFcFEVtQ0cOFvMntNF7M4o4sDZEmobLAAkhPuSHB/CFfHGSBpZosB5SJeLEILahkZSs0rZfbqYPRlF7M0oosz6JeuQfn7NI2iuGBQqC4z1YhLoQoiLNFo0x3KsX7KmF7LndBHV9Y2YFCTGBjEtIZRpg8OYMDAYL3f5krW3kEAXQrSrrsHCgbPFbE8v5Ou0Ag6eK6HBovFwMzFhQDDTBocyMS6ExJhAGSbpQBLoQojLVlHbwJ7TRc1X8Mesi4yZFAyN8GdsbBBJA4IYGxvE0Ag/WWzMTiTQhRBdVlxZx8HMEg6eLSEls4SD50ooqaoHwNvdTGJMIEkDgkjqH8TY/kFEB3rJejQ9QIYtCiG6LNjXg5nD+jFzWD/AmMl6prCKlMwSDlhD/u9fZ1BnHUkT5ufJ2NhAEmMDGRMbSGJMEOH+MlyyJ0mgCyE6RSlFXJgvcWG+XJcUAxj98Mdzyzh4zriCP5xZyn+/yaOpIyAq0IvEGCPgx8QGkRgTSLCvhwPPwrVIoAshuo2Hm4kxsUGMiQ3ie1OMfZW1DRzJLuNQZgmHs0o5nFnKpqPnm1/TP8SbMTFBjIoJYERUACMiA4gI8JTumk6QQBdC9Chf6808kuNDmveV1dSTag33Q1mlHMosYcPhnObjwT7ujIgKYHhkACOi/BkRFcDgfn4yfLIdEuhCCLsL8HJnakIYUxPCmveVVtdzPKeM47nlHMsp41hOGf/YfYaaeqNP3mxSJIT7Ngf9qOgA6bJpRQJdCNErBHq7c8UgY6Zqk0aLJqOwkuM5F0J+b0YxnxzMbm4TG+zd/KVrYkwgiTGBBPr0zZmuEuhCiF7LuCr3IyHcjwVjopr3l1bVcyTb6K45nFnK4axSNh7ObT4+IMSHxFgj3MfEBDIqJrBPLGcggS6EcDqBPu5MHRzG1MEXumxKquqML12tIZ9yroQNhy70yw+L8GdSfDCT4oxFySIDXW9BMplYJIRwWUWVRsinnCthT0YR+88UU1nXCBija5LjQkmODyY5PpS4UB+nGFkjM0WFEALjrk9Hc8rYfbpp3fhiiirrAGMiVHJ8MMlxIUyMC2FohD8ebr1vOQMJdCGEuISm+7Y2LSm8+3QRWSXVAHiYTQyL9Gd0TCCjY4wRNUMj/B0+dFICXQghOiirpJr9Z4pJzS4lNauU1KwySquNNWvcTIqhEf7NAT8qJpCRUQF2DXkJdCGE6CStNZnF1cZEqKxSUrPLSM0qbe6qMZsUg8J8GRbpbx0j78+wSH9igrx7pE9eFucSQohOUkrRP8SH/iE+zEs0hk5qrckprTECPquUYznlHDxXwvoWo2r8Pd0YFunP8Ch/hkUGMCLSn6GR/gR49dzwSQl0IYS4TEopooO8iQ7yZs6oyOb95TX1nDhfzvHcco7nlPNNbjmfHMymvOZsc5uYIG9+NXdY84Jm3UkCXQghuom/lzsTBoYwYeCFdWuaruaP55Y1B324X88sI9xuoCulvICvAE9r+9Va68dbtfEE3gYmAIXAzVrrjG6vVgghnEzLq/lZwyN69Hd1ZJBlLTBLaz0WSALmKqUmt2pzN1CstR4MPAv8oXvLFEII0Z52A10bKqw/ulsfrYfGXAe8Zd1eDcxWzjDlSgghXEiHpkEppcxKqYNAHvC51npXqyYxwDkArXUDUAqEtmqDUmqFUmqvUmpvfn5+1yoXQgjxLR0KdK11o9Y6CYgFkpVSozvzy7TWr2mtJ2qtJ4aHh3fmLYQQQrThshYq0FqXAF8Ac1sdygL6Ayil3IBAjC9HhRBC2Em7ga6UCldKBVm3vYFrgOOtmq0F7rRu3wT8VztqCqoQQvRRHRmHHgW8pZQyY/wB+EBrvV4p9QSwV2u9FngTeEcplQYUAbf0WMVCCCEuqd1A11ofAsZdYv9jLbZrgO92b2lCCCEuh8MW51JK5QNnOvnyMKCgG8vpzfrKufaV84S+c6595TzBvuc6UGt9yVElDgv0rlBK7W1rtTFX01fOta+cJ/Sdc+0r5wm951x73+04hBBCdIoEuhBCuAhnDfTXHF2AHfWVc+0r5wl951z7ynlCLzlXp+xDF0IIcTFnvUIXQgjRigS6EEK4CKcLdKXUXKXUN0qpNKXUw46up6copTKUUoeVUgeVUi51N22l1EqlVJ5SKrXFvhCl1OdKqZPW52BH1thd2jjX/1VKZVk/24NKqfmOrLE7KKX6K6W+UEodVUodUUo9YN3vUp+rjfPsFZ+pU/WhW5cfOIGxnkwmsAe4VWt91KGF9QClVAYwUWvtchMzlFJXARXA21rr0dZ9/w8o0lo/bf1DHay1fsiRdXaHNs71f4EKrfWfHFlbd1JKRQFRWuv9Sil/YB9wPbAcF/pcbZznUnrBZ+psV+jJQJrW+pTWug54H+PmGsKJaK2/wljzp6WWN0l5C+M/EqfXxrm6HK11jtZ6v3W7HDiGcZ8El/pcbZxnr+Bsgd58Iw2rTHrR/5jdTAOblFL7lFIrHF2MHURorXOs27lAz9580fF+rJQ6ZO2ScepuiNaUUnEY6z/twoU/11bnCb3gM3W2QO9LrtRajwfmAfdb/+neJ1iXXnaevsDL9wqQgHGP3hzgz44tp/sopfyANcDPtNZlLY+50ud6ifPsFZ+pswV68400rGKt+1yO1jrL+pwHfIzR3eTKzlv7J5v6KfMcXE+P0Vqft94FzAK8jot8tkopd4yQe1dr/ZF1t8t9rpc6z97ymTpboO8Bhiil4pVSHhjrrq91cE3dTinla/3CBaWUL3AtkGr7VU6v5U1S7gQ+cWAtPaop4KxuwAU+W+tN4d8Ejmmtn2lxyKU+17bOs7d8pk41ygXAOhzoOcAMrNRa/5+DS+p2SqlBGFflYKxZ/w9XOk+l1HvADIwlR88DjwP/Aj4ABmAsq7xUa+30Xya2ca4zMP5proEM4N4W/cxOSSl1JbAVOAxYrLt/jdG/7DKfq43zvJVe8Jk6XaALIYS4NGfrchFCCNEGCXQhhHAREuhCCOEiJNCFEMJFSKALIYSLkEAXQggXIYEuhBAu4v8DWxM94vFkonUAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zV0t4mnpSBR5"
      },
      "source": [
        "reverse_target_word_index=y_tokenizer.index_word\n",
        "reverse_source_word_index=x_tokenizer.index_word\n",
        "target_word_index=y_tokenizer.word_index"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BIEmURNuSCIy"
      },
      "source": [
        "# Encode the input sequence to get the feature vector\n",
        "encoder_model = Model(inputs=encoder_inputs,outputs=[encoder_outputs, state_h, state_c])\n",
        "\n",
        "# Decoder setup\n",
        "# Below tensors will hold the states of the previous time step\n",
        "decoder_state_input_h = Input(shape=(latent_dim,))\n",
        "decoder_state_input_c = Input(shape=(latent_dim,))\n",
        "decoder_hidden_state_input = Input(shape=(max_text_len,latent_dim))\n",
        "\n",
        "# Get the embeddings of the decoder sequence\n",
        "dec_emb2 = dec_emb_layer(decoder_inputs) \n",
        "# To predict the next word in the sequence, set the initial states to the states from the previous time step\n",
        "decoder_outputs2, state_h2, state_c2 = decoder_lstm(dec_emb2, initial_state=[decoder_state_input_h, decoder_state_input_c])\n",
        "\n",
        "# A dense softmax layer to generate prob dist. over the target vocabulary\n",
        "decoder_outputs2 = decoder_dense(decoder_outputs2) \n",
        "\n",
        "# Final decoder model\n",
        "decoder_model = Model(\n",
        "    [decoder_inputs] + [decoder_hidden_state_input,decoder_state_input_h, decoder_state_input_c],\n",
        "    [decoder_outputs2] + [state_h2, state_c2])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cTfeovYASH3y"
      },
      "source": [
        "def decode_sequence(input_seq):\n",
        "    # Encode the input as state vectors.\n",
        "    e_out, e_h, e_c = encoder_model.predict(input_seq)\n",
        "    \n",
        "    # Generate empty target sequence of length 1.\n",
        "    target_seq = np.zeros((1,1))\n",
        "    \n",
        "    # Populate the first word of target sequence with the start word.\n",
        "    target_seq[0, 0] = target_word_index['sostok']\n",
        "\n",
        "    stop_condition = False\n",
        "    decoded_sentence = ''\n",
        "    while not stop_condition:\n",
        "      \n",
        "        output_tokens, h, c = decoder_model.predict([target_seq] + [e_out, e_h, e_c])\n",
        "\n",
        "        # Sample a token\n",
        "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
        "        sampled_token = reverse_target_word_index[sampled_token_index]\n",
        "        \n",
        "        if(sampled_token!='eostok'):\n",
        "            decoded_sentence += ' '+sampled_token\n",
        "\n",
        "        # Exit condition: either hit max length or find stop word.\n",
        "        if (sampled_token == 'eostok'  or len(decoded_sentence.split()) >= (max_summary_len-1)):\n",
        "            stop_condition = True\n",
        "\n",
        "        # Update the target sequence (of length 1).\n",
        "        target_seq = np.zeros((1,1))\n",
        "        target_seq[0, 0] = sampled_token_index\n",
        "\n",
        "        # Update internal states\n",
        "        e_h, e_c = h, c\n",
        "\n",
        "    return decoded_sentence"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hbKKzJaCSOii"
      },
      "source": [
        "def seq2summary(input_seq):\n",
        "    newString=''\n",
        "    for i in input_seq:\n",
        "        if((i!=0 and i!=target_word_index['sostok']) and i!=target_word_index['eostok']):\n",
        "            newString=newString+reverse_target_word_index[i]+' '\n",
        "    return newString\n",
        "\n",
        "def seq2text(input_seq):\n",
        "    newString=''\n",
        "    for i in input_seq:\n",
        "        if(i!=0):\n",
        "            newString=newString+reverse_source_word_index[i]+' '\n",
        "    return newString"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ULWhcClrUqXy"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GiLhVgbyUzxf",
        "outputId": "6cebd306-07a6-4708-dfef-14025a98289c"
      },
      "source": [
        "import pickle\n",
        "\n",
        "model.save('news_pre.h5')\n",
        "\n",
        "with open('y_tokenizer.pickle', 'wb') as handle:\n",
        "    pickle.dump(y_tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
        "  \n",
        "with open('x_tokenizer.pickle', 'wb') as handle:\n",
        "    pickle.dump(x_tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
        "\n",
        "\n",
        "encoder_model.save('encoder.h5')\n",
        "decoder_model.save('decoder.h5')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n"
          ]
        }
      ]
    }
  ]
}