Airline_Sentiment_Analysis_Using_LSTMs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "id": "INsvANBUQ_Nn"
   },
   "outputs": [],
   "source": [
    "\n",
    "#importing libraries for model evaluation and algorithms\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import re\n",
    "import collections\n",
    "import nltk\n",
    "from sklearn import preprocessing\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "# Packages for data preparation\n",
    "from sklearn.model_selection import train_test_split\n",
    "from nltk.corpus import stopwords\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "#dl libraraies\n",
    "import keras\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Conv2D , MaxPool2D , Flatten , Dropout , BatchNormalization,Reshape,Dot,Concatenate,Add,Lambda\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from tensorflow.keras.callbacks import ReduceLROnPlateau\n",
    "import cv2\n",
    "from tensorflow.keras.optimizers import Adam,SGD,Adagrad,Adadelta,RMSprop\n",
    "import os\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.regularizers import l2\n",
    "from keras.layers.recurrent import LSTM\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "# specifically for deeplearning.\n",
    "from tensorflow.keras.layers import Dropout, Flatten,Activation,Input,Embedding\n",
    "from tensorflow.keras.layers import Conv2D, MaxPooling2D, BatchNormalization\n",
    "# Packages for modeling\n",
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import regularizers\n",
    "from kerastuner import RandomSearch\n",
    "from kerastuner.engine.hyperparameters import HyperParameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 445
    },
    "id": "DoU4sypRQ_Gj",
    "outputId": "909ca2ef-2ef1-4d23-8bc7-12f499d25930"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>airline_sentiment</th>\n",
       "      <th>airline_sentiment_confidence</th>\n",
       "      <th>negativereason</th>\n",
       "      <th>negativereason_confidence</th>\n",
       "      <th>airline</th>\n",
       "      <th>airline_sentiment_gold</th>\n",
       "      <th>name</th>\n",
       "      <th>negativereason_gold</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>text</th>\n",
       "      <th>tweet_coord</th>\n",
       "      <th>tweet_created</th>\n",
       "      <th>tweet_location</th>\n",
       "      <th>user_timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>570306133677760513</td>\n",
       "      <td>neutral</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cairdin</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica What @dhepburn said.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:35:52 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Eastern Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>570301130888122368</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.3486</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica plus you've added commercials t...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:59 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>570301083672813571</td>\n",
       "      <td>neutral</td>\n",
       "      <td>0.6837</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yvonnalynn</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica I didn't today... Must mean I n...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:48 -0800</td>\n",
       "      <td>Lets Play</td>\n",
       "      <td>Central Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>570301031407624196</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Bad Flight</td>\n",
       "      <td>0.7033</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica it's really aggressive to blast...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:15:36 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>570300817074462722</td>\n",
       "      <td>negative</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Can't Tell</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>Virgin America</td>\n",
       "      <td>NaN</td>\n",
       "      <td>jnardino</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>@VirginAmerica and it's a really big bad thing...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-02-24 11:14:45 -0800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pacific Time (US &amp; Canada)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id  ...               user_timezone\n",
       "0  570306133677760513  ...  Eastern Time (US & Canada)\n",
       "1  570301130888122368  ...  Pacific Time (US & Canada)\n",
       "2  570301083672813571  ...  Central Time (US & Canada)\n",
       "3  570301031407624196  ...  Pacific Time (US & Canada)\n",
       "4  570300817074462722  ...  Pacific Time (US & Canada)\n",
       "\n",
       "[5 rows x 15 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#reading the dataframe\n",
    "df=pd.read_csv('Tweets.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DGb7zeFpwSc4",
    "outputId": "dd941019-e245-45b4-a10b-1aca63d14a2e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    9178\n",
       "1    2363\n",
       "Name: Sentiment, dtype: int64"
      ]
     },
     "execution_count": 55,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#removing the neutral sentiments considering that positive and negative sentiments matter more \n",
    "df=df[df['airline_sentiment']!='neutral']\n",
    "#now since neutral elements are deleted so I need to reset the indices\n",
    "df.reset_index(inplace=True,drop=True)\n",
    "#positive sentiments to 1 and negative to 0 \n",
    "def partition(x):\n",
    "    if x =='positive':\n",
    "        return 1\n",
    "    return 0\n",
    "actualSentiment = df['airline_sentiment']\n",
    "positiveNegative = actualSentiment.map(partition) \n",
    "df['Sentiment'] = positiveNegative\n",
    "df['Sentiment'].value_counts() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 462
    },
    "id": "Wfbovcn8wSfu",
    "outputId": "81904257-4d52-42e5-9507-36e060aac812"
   },
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "id": "SoDUJUsPU0M3"
   },
   "outputs": [],
   "source": [
    "#Setting parameters which will be used throughout\n",
    "num_words = 15000  # Parameter indicating the number of words we'll put in the dictionary\n",
    "val_size = 1000  # Size of the validation set\n",
    "epochs = 20  # Number of epochs we usually start to train with\n",
    "batch_size = 512  # Size of the batches used in the mini-batch gradient descent\n",
    "#Taking only two columns since it's a sentiment analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "id": "iJTh5-ShU0P2"
   },
   "outputs": [],
   "source": [
    "#tweets conssits of every document as an array of tokenized words which are later appended to docs \n",
    "tweets=[word_tokenize(tweet) for tweet in df['text']]\n",
    "docs=[]\n",
    "for j in range(0,len(tweets)):\n",
    "    docs.append(tweets[j])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "id": "F-rXkF_1U0iE"
   },
   "outputs": [],
   "source": [
    "#stops included both the stopwords and punctuations\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "stops = set(stopwords.words('english'))\n",
    "punctuations = list(string.punctuation)\n",
    "not_list = [\"n't\", \"not\", \"no\"]\n",
    "stops.update(punctuations)\n",
    "stops.update(not_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "id": "1mxGxUnBWSv3"
   },
   "outputs": [],
   "source": [
    "#to get the simple pos(part of speech) tag\n",
    "from nltk.corpus import wordnet\n",
    "def get_simple_pos(tag):\n",
    "    if tag.startswith('J'):\n",
    "        return wordnet.ADJ\n",
    "    elif tag.startswith('V'):\n",
    "        return wordnet.VERB\n",
    "    elif tag.startswith('N'):\n",
    "        return wordnet.NOUN\n",
    "    elif tag.startswith('R'):\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return wordnet.NOUN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "id": "_nO4zHxZWSzJ"
   },
   "outputs": [],
   "source": [
    "#to get the pos tag for a word\n",
    "from nltk import pos_tag\n",
    "# now we are going to clean our data \n",
    "# we will remove stopwords and punctuations and lemmatize each document\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "lemmatizer=WordNetLemmatizer()\n",
    "def clean(words):\n",
    "    output=[]\n",
    "    for word in words:\n",
    "        if word.lower() not in stops or word.lower() in not_list:\n",
    "            pos=pos_tag(word)\n",
    "            clean_word=lemmatizer.lemmatize(word,pos=get_simple_pos(pos[0][1]))\n",
    "            output.append(clean_word.lower())\n",
    "    str1=\" \".join(output).encode('utf-8')        \n",
    "    return str1\n",
    "docs=[ clean(doc) for doc in docs]      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 479
    },
    "id": "HEdvYfQdWS1m",
    "outputId": "b062ab67-1039-460b-ab75-fc8c2ea53c4a"
   },
   "outputs": [],
   "source": [
    "#docs was the cleaned lemmatized text which has been appended into the dataframe for further use\n",
    "df['CleanedText']=docs\n",
    "df['CleanedText']=df['CleanedText'].str.decode(\"utf-8\")\n",
    "def remove_mentions(input_text):\n",
    "        return re.sub(r'@\\w+', '', input_text)\n",
    "df.text = df.CleanedText.apply(remove_mentions)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "id": "eywgfErHwShp",
    "outputId": "b9bbcb3b-e642-4825-8c7c-2775ec897e5b"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CleanedText</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>virginamerica plus 've added commercial experi...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>virginamerica 's really aggressive blast obnox...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>virginamerica 's really big bad thing</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>virginamerica seriously would pay 30 flight se...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>virginamerica yes nearly every time fly vx “ e...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         CleanedText  Sentiment\n",
       "0  virginamerica plus 've added commercial experi...          1\n",
       "1  virginamerica 's really aggressive blast obnox...          0\n",
       "2              virginamerica 's really big bad thing          0\n",
       "3  virginamerica seriously would pay 30 flight se...          0\n",
       "4  virginamerica yes nearly every time fly vx “ e...          1"
      ]
     },
     "execution_count": 64,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#taking only two columns in the dataframe\n",
    "df=df[['CleanedText','Sentiment']]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "id": "_B9KW6XjAVa_"
   },
   "outputs": [],
   "source": [
    "#taking variables to be used for train test split as X,y\n",
    "X,Y=df['CleanedText'].values,pd.get_dummies(df.Sentiment.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "e57a2yp9AVfN",
    "outputId": "6c99461e-92f8-491c-cc38-bd8b006bf204"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitted tokenizer on 11541 documents\n",
      "15000 words in dictionary\n",
      "Top 5 most common words are: [('united', 3426), ('flight', 3321), ('usairways', 2651), ('americanair', 2465), (\"n't\", 1878)]\n"
     ]
    }
   ],
   "source": [
    "#using tokenizers to create the tokens having no of words=15000(num_words)\n",
    "tk = Tokenizer(num_words=num_words,\n",
    "               filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n",
    "               lower=True,\n",
    "               split=\" \")\n",
    "#Complete data is tokenized to vectors and padding is done using zeros to match its length to the largest text in the dataset.\n",
    "tk.fit_on_texts(X)\n",
    "X = tk.texts_to_sequences(X)\n",
    "X = pad_sequences(X)\n",
    "#print(X[:2])\n",
    "print('Fitted tokenizer on {} documents'.format(tk.document_count))\n",
    "print('{} words in dictionary'.format(tk.num_words))\n",
    "print('Top 5 most common words are:', collections.Counter(tk.word_counts).most_common(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8CeCPtjnUWIl",
    "outputId": "2c0909b6-3b76-4f2f-d9a9-c46ed9296818"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# Train data samples: (9232, 27)\n",
      "# Test data samples: (2309, 27)\n"
     ]
    }
   ],
   "source": [
    "#train test split\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size=0.2, random_state=42)\n",
    "print('# Train data samples:', X_train.shape)\n",
    "print('# Test data samples:', X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hndF1pitCISi",
    "outputId": "c73705fa-f623-4807-99a4-4de28d4dc0be"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of validation set: (924, 27)\n"
     ]
    }
   ],
   "source": [
    "#getting validation data as a part of training data\n",
    "X_train_rest, X_valid, Y_train_rest, Y_valid = train_test_split(X_train,Y_train, test_size=0.1, random_state=37)\n",
    "print('Shape of validation set:',X_valid.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "id": "EnlWWTk-E4D4"
   },
   "outputs": [],
   "source": [
    "#Function defined to test the models in the test set\n",
    "def test_model(model, epoch_stop):\n",
    "    model.fit(X_test\n",
    "              , Y_test\n",
    "              , epochs=epoch_stop\n",
    "              , batch_size=batch_size\n",
    "              , verbose=0)\n",
    "    results = model.evaluate(X_test, Y_test)\n",
    "    \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "h1kMJUhJOnDL",
    "outputId": "7c24300b-6224-4202-d11a-ce8210f77609"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 27, 128)           1920000   \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 196)               254800    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 32)                6304      \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 2)                 66        \n",
      "=================================================================\n",
      "Total params: 2,181,170\n",
      "Trainable params: 2,181,170\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "embed_dim = 128 #dimension of the word embedding vector for each word in a sequence \n",
    "lstm_out = 196  #no of lstm layers\n",
    "lstm_model = Sequential()\n",
    "lstm_model.add(Embedding(num_words, embed_dim,input_length = X_train.shape[1]))\n",
    "#Adding dropout\n",
    "lstm_model.add(LSTM(lstm_out, dropout=0.2, recurrent_dropout=0.2))\n",
    "#Adding a regularized dense layer\n",
    "lstm_model.add(layers.Dense(32,kernel_regularizer=regularizers.l2(0.001),activation='relu'))\n",
    "lstm_model.add(layers.Dropout(0.5))\n",
    "lstm_model.add(Dense(2,activation='softmax'))\n",
    "lstm_model.compile(loss = 'categorical_crossentropy', optimizer='adam',metrics = ['accuracy'])\n",
    "print(lstm_model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rnaNvVukOnIx",
    "outputId": "67920049-638a-4128-adb3-96a97db7cc59"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/4\n",
      "17/17 [==============================] - 15s 728ms/step - loss: 0.6842 - accuracy: 0.7224 - val_loss: 0.4905 - val_accuracy: 0.7955\n",
      "Epoch 2/4\n",
      "17/17 [==============================] - 12s 703ms/step - loss: 0.4640 - accuracy: 0.8102 - val_loss: 0.3629 - val_accuracy: 0.8582\n",
      "Epoch 3/4\n",
      "17/17 [==============================] - 12s 700ms/step - loss: 0.3267 - accuracy: 0.8870 - val_loss: 0.2661 - val_accuracy: 0.9123\n",
      "Epoch 4/4\n",
      "17/17 [==============================] - 12s 702ms/step - loss: 0.1965 - accuracy: 0.9515 - val_loss: 0.2216 - val_accuracy: 0.9253\n"
     ]
    }
   ],
   "source": [
    "#model trained on the training data and taking validation data into account to avoid overfitting for 4 epochs \n",
    "history_LSTM=lstm_model.fit(X_train_rest, Y_train_rest, epochs = 4, batch_size=batch_size,validation_data=(X_valid, Y_valid),verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GgOA7WOIeQSZ",
    "outputId": "299b2f20-9fae-4390-ebbf-dc2d11b060e2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "73/73 [==============================] - 1s 16ms/step - loss: 0.1254 - accuracy: 0.9688\n",
      "/n\n",
      "Test accuracy of lstm model: 96.88%\n"
     ]
    }
   ],
   "source": [
    "#prediction by our lstm model on the test dataset\n",
    "lstm_results = test_model(lstm_model, 3)\n",
    "print('/n')\n",
    "print('Test accuracy of lstm model: {0:.2f}%'.format(lstm_results[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "id": "D48NAU0nXEcR"
   },
   "outputs": [],
   "source": [
    "#building the model using Keras Tuner so taking a range of values for the layers desired to be tuned.\n",
    "def build_model(hp):  \n",
    "  tuned_lstm_model = keras.Sequential([     \n",
    "     # giving range of values for output_dim for tuning for Embedding layer                                  \n",
    "     keras.layers.Embedding(\n",
    "        input_dim=num_words, \n",
    "        output_dim=hp.Int('embeddings', min_value=32, max_value=128, step=16),\n",
    "        input_length = X_train.shape[1]\n",
    "    ),      \n",
    "      # giving range of values for no. of units for tuning for LSTM layer\n",
    "      keras.layers.LSTM(\n",
    "        units=hp.Int('lstm_1_units', min_value=64, max_value=224,step=32), \n",
    "        dropout=hp.Int('drop_1_units', min_value=0, max_value=8,step=2)/10,\n",
    "        recurrent_dropout=0.0\n",
    "    ),                                                  \n",
    "    # giving range of values for no. of units for tuning for dense layer\n",
    "    keras.layers.Dense(\n",
    "        units=hp.Int('dense_2_units', min_value=32, max_value=128, step=16),\n",
    "        activation='relu',\n",
    "        kernel_regularizer=regularizers.l2(0.001)\n",
    "    ),   \n",
    "    # giving range of values for rate for tuning for dropout layer\n",
    "    keras.layers.Dropout(\n",
    "        rate=hp.Int('drop_1_rate', min_value=1, max_value=8, step=1)/10), \n",
    "    keras.layers.Dense(2, activation='softmax')\n",
    "  ])\n",
    "  #model compiled using Adam's optimizer with choices for learning rate\n",
    "  tuned_lstm_model.compile(optimizer=keras.optimizers.Adam(hp.Choice('learning_rate', values=[1e-2, 1e-3])),\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "  \n",
    "  return tuned_lstm_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "id": "Ww9khaEBYfK9"
   },
   "outputs": [],
   "source": [
    "#A function defined for keras tuner to look at each of the models\n",
    "tuner_search=RandomSearch(build_model,\n",
    "                          objective='val_accuracy',\n",
    "                          max_trials=5,directory='dir',\n",
    "    project_name='lstm_sentiment_Analysis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yvETFy5PYfP1",
    "outputId": "1d718a1d-dcc2-48b9-97c4-a1fc6eb251da"
   },
   "outputs": [],
   "source": [
    "#function called to search the results for different combinations of the parameter values in layers\n",
    "tuner_search.search(X_train_rest,Y_train_rest,epochs=3,validation_data=(X_valid,Y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "L6YWjAFGYfRh",
    "outputId": "547730fa-d269-44ca-87f5-d650073319a9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 27, 48)            720000    \n",
      "_________________________________________________________________\n",
      "lstm (LSTM)                  (None, 160)               133760    \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 96)                15456     \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 96)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 2)                 194       \n",
      "=================================================================\n",
      "Total params: 869,410\n",
      "Trainable params: 869,410\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#to get the best model parameters\n",
    "tuned_lstm_model=tuner_search.get_best_models(num_models=1)[0]\n",
    "tuned_lstm_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9m_m1UbQdXTt",
    "outputId": "d3339b72-8050-491c-b711-cd5df61115f0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/4\n",
      "17/17 [==============================] - 12s 701ms/step - loss: 0.1209 - accuracy: 0.9693 - val_loss: 0.2269 - val_accuracy: 0.9242\n",
      "Epoch 2/4\n",
      "17/17 [==============================] - 12s 694ms/step - loss: 0.1000 - accuracy: 0.9763 - val_loss: 0.2243 - val_accuracy: 0.9264\n",
      "Epoch 3/4\n",
      "17/17 [==============================] - 12s 702ms/step - loss: 0.0713 - accuracy: 0.9863 - val_loss: 0.2507 - val_accuracy: 0.9264\n",
      "Epoch 4/4\n",
      "17/17 [==============================] - 12s 690ms/step - loss: 0.0548 - accuracy: 0.9901 - val_loss: 0.3078 - val_accuracy: 0.9351\n"
     ]
    }
   ],
   "source": [
    "#model trained on the training data and taking validation data into account to avoid overfitting for 4 epochs \n",
    "history_LSTM=lstm_model.fit(X_train_rest, Y_train_rest, epochs = 4, batch_size=batch_size,validation_data=(X_valid, Y_valid),verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Ed-W3dZ4dBd3",
    "outputId": "81b3d1e0-e7ad-4ca2-ff87-587b67a56301"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "73/73 [==============================] - 1s 14ms/step - loss: 0.0694 - accuracy: 0.9809\n",
      "/n\n",
      "Test accuracy of lstm model: 98.09%\n"
     ]
    }
   ],
   "source": [
    "#prediction by our tuned lstm model on the test dataset. This accuracy is better than the untuned model\n",
    "lstm_results = test_model(tuned_lstm_model, 5)\n",
    "print('/n')\n",
    "print('Test accuracy of lstm model: {0:.2f}%'.format(lstm_results[1]*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kk26OATDiXXv"
   },
   "source": [
    "##Now we will see the predictions by our model on sample movie reviews"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UGinF3USilsL"
   },
   "source": [
    "***Class 1 is positive class and class 0 is negative class***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AQwynIG4HyeL",
    "outputId": "affbe939-a7bd-4c3c-de51-cfd9f8959e68"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 10)"
      ]
     },
     "execution_count": 81,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#First the sample text is a positive review, so lets see its prediction by our model\n",
    "sample_text = ('The movie was good. Everything was so awesome. Highly recommended.')\n",
    "#converting into sequence because my model is trained on such a sequence\n",
    "sample = tk.texts_to_sequences([sample_text])\n",
    "sample = pad_sequences(sample)\n",
    "sample.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fbjpJ6yCZ5mu",
    "outputId": "aa2e49b7-07a1-494e-a9b3-bc066fa20904"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,\n",
       "           0.,    0.,    0.,    0.,    0.,    0.,    0.,    0., 2423.,\n",
       "        1348., 7534.,   89.,  456., 7534., 2175.,  190., 2243., 3829.]])"
      ]
     },
     "execution_count": 82,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#In training 27 was the no of tokens for each tweet so padding by zeros so that model can accept it\n",
    "k=np.zeros((1,27))\n",
    "k[0,-10:]=sample\n",
    "k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "w8MIEpNnHygz",
    "outputId": "fd7fa493-e718-419a-bfe1-acbf9f3fa845"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\n",
      "[0.16331808 0.8366819 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
      "  warnings.warn('`model.predict_classes()` is deprecated and '\n"
     ]
    }
   ],
   "source": [
    "#Prediction by model and as expected we get the class as 1 by almost 83%\n",
    "predictions = lstm_model.predict_classes(np.array(k))\n",
    "prediction_probability=lstm_model.predict(np.array(k))\n",
    "print(predictions)\n",
    "print(prediction_probability[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "NiqGImrlevsV",
    "outputId": "e6e66147-7b48-4034-e995-0762a2cb2240"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
      "  warnings.warn('`model.predict_classes()` is deprecated and '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\n",
      "[0.04985565 0.95014435]\n"
     ]
    }
   ],
   "source": [
    "#Prediction by tuned model and as expected we get the class as 1 by almost 95%\n",
    "predictions = tuned_lstm_model.predict_classes(np.array(k))\n",
    "prediction_probability=tuned_lstm_model.predict(np.array(k))\n",
    "print(predictions)\n",
    "print(prediction_probability[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "XRO6SxWxQ8l3",
    "outputId": "6f759af1-705b-4ce5-92ba-ca7f2564b943"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 13)"
      ]
     },
     "execution_count": 86,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Secondly the sample text is a negative review, so lets see its prediction by our model\n",
    "sample_text = ('The movie was pathetic and terrible.Nothing was making any sense. Highly unacceptable.')\n",
    "#converting into sequence because my model is trained on such a sequence\n",
    "sample = tk.texts_to_sequences([sample_text])\n",
    "sample = pad_sequences(sample)\n",
    "sample.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0M84WOeQOS4j",
    "outputId": "89a2245e-5bbe-450e-80fe-7ca266aae7ca"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[    0.,     0.,     0.,     0.,     0.,     0.,     0.,     0.,\n",
       "            0.,     0.,     0.,     0.,     0.,     0.,  2423.,  1348.,\n",
       "         7534.,   845.,  1778.,   223.,   153.,  7534.,   228., 12070.,\n",
       "          656.,  2243.,   257.]])"
      ]
     },
     "execution_count": 88,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#In training 27 was the no of tokens for each tweet so padding by zeros so that model can accept it\n",
    "k=np.zeros((1,27))\n",
    "k[0,-13:]=sample\n",
    "k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Ddzv2kjfj-kf",
    "outputId": "cff47d1b-18de-4919-96a1-f21b94e54875"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\n",
      "[9.9998820e-01 1.1783306e-05]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
      "  warnings.warn('`model.predict_classes()` is deprecated and '\n"
     ]
    }
   ],
   "source": [
    "#Prediction by model and as expected we get the class as 0 by 99.99% probability. \n",
    "predictions = lstm_model.predict_classes(np.array(k))\n",
    "prediction_probability=lstm_model.predict(np.array(k))\n",
    "print(predictions)\n",
    "print(prediction_probability[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "HGejIeFyfDuK",
    "outputId": "7c7b7b87-eafb-4320-a9df-2b17d60e172d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\n",
      "[9.999629e-01 3.712654e-05]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
      "  warnings.warn('`model.predict_classes()` is deprecated and '\n"
     ]
    }
   ],
   "source": [
    "#Prediction by tuned model and as expected we get the class as 0 by 99.99% probability. \n",
    "predictions = tuned_lstm_model.predict_classes(np.array(k))\n",
    "prediction_probability=tuned_lstm_model.predict(np.array(k))\n",
    "print(predictions)\n",
    "print(prediction_probability[0])"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Airline Sentiment Analysis Using LSTMs.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}