analytics_vidhya_snli_discodisco_simplebert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#seeding for comparing experiment in part 2\n",
    "import torch\n",
    "SEED = 1111\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SNLI Bert\n",
    "## Second Tutorial\n",
    "https://towardsdatascience.com/fine-tuning-pre-trained-transformer-models-for-sentence-entailment-d87caf9ec9db\n",
    "Check his Github code for complete notebook. I never referred to it. Medium was enough.\n",
    "BERT in keras-tf: https://towardsdatascience.com/bert-in-keras-with-tensorflow-hub-76bcbc9417b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define macros\n",
    "BERT_MODEL = 'bert-base-uncased'\n",
    "MAX_SEQ_LENGTH = 100 # we dont need to enforce this now because snli is a relatively sanitized dataset where sentence lenghts are reasonable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://nlp.stanford.edu/projects/snli/snli_1.0.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13897"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# custom reader needed to handle quotechars\n",
    "def read_df_custom(file):\n",
    "    header = 'doc     unit1_toks      unit2_toks      unit1_txt       unit2_txt       s1_toks s2_toks unit1_sent      unit2_sent      dir     nuc_children    sat_children    genre   u1_discontinuous        u2_discontinuous       u1_issent        u2_issent       u1_length       u2_length       length_ratio    u1_speaker      u2_speaker      same_speaker    u1_func u1_pos  u1_depdir       u2_func u2_pos  u2_depdir       doclen  u1_position      u2_position     percent_distance        distance        lex_overlap_words       lex_overlap_length      unit1_case      unit2_case      label'\n",
    "    header = header.split()\n",
    "    df = pd.DataFrame(columns=['unit1_txt', 'unit1_sent', 'unit2_txt', 'unit2_sent', 'dir', 'label'])\n",
    "    file = open(file, 'r')\n",
    "\n",
    "    rows = []\n",
    "    count = 0 \n",
    "    for line in file:\n",
    "        line = line[:-1].split('\\t')\n",
    "        count+=1\n",
    "        if count ==1: continue\n",
    "        row = {}\n",
    "        for column in ['unit1_txt', 'unit1_sent', 'unit2_txt', 'unit2_sent', 'dir', 'label']:\n",
    "            index = header.index(column)\n",
    "            row[column] = line[index]\n",
    "        rows.append(row)\n",
    "\n",
    "    df = pd.concat([df, pd.DataFrame.from_records(rows)])\n",
    "    return df\n",
    "\n",
    "# we only need specific columns\n",
    "train_df = read_df_custom('./eng.rst.gum_train_enriched.rels')\n",
    "test_df = read_df_custom('./eng.rst.gum_test_enriched.rels')\n",
    "val_df = read_df_custom('./eng.rst.gum_dev_enriched.rels')\n",
    "len(train_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dropping any empty values\n",
    "train_df.dropna(inplace=True)\n",
    "val_df.dropna(inplace=True)\n",
    "test_df.dropna(inplace=True)\n",
    "\n",
    "# train_df = train_df[:1000]\n",
    "# val_df = val_df[:100]\n",
    "# test_df = test_df[:100]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare a dataset handler class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import torch\n",
    "from torch.utils.data import Dataset, TensorDataset, DataLoader\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "from transformers import BertTokenizer\n",
    "import pandas as pd\n",
    "\n",
    "class MNLIDataBert(Dataset):\n",
    "\n",
    "  def __init__(self, train_df, val_df, test_df):\n",
    "    self.train_df = train_df\n",
    "    self.val_df = val_df\n",
    "    self.test_df = test_df\n",
    "\n",
    "    self.tokenizer = BertTokenizer.from_pretrained(BERT_MODEL, do_lower_case=True) # Using a pre-trained BERT tokenizer to encode sentences\n",
    "    self.train_data = None\n",
    "    self.val_data = None\n",
    "    self.test_data = None\n",
    "    self.init_data()\n",
    "\n",
    "  def init_data(self):\n",
    "    self.get_label_mapping()\n",
    "    self.train_data = self.load_data(self.train_df)\n",
    "    self.val_data = self.load_data(self.val_df)\n",
    "    self.test_data = self.load_data(self.test_df)\n",
    "\n",
    "  def get_label_mapping(self):\n",
    "    labels = {}\n",
    "    labels_list = list(set(list(self.train_df['label'].unique()) + list(self.test_df['label'].unique()) + list(self.val_df['label'].unique())))\n",
    "    for i in range(len(labels_list)):\n",
    "        labels[labels_list[i]] = i\n",
    "    self.label_dict = labels# {'entailment': 0, 'contradiction': 1, 'neutral': 2}\n",
    "    # needed later for classification report object to generate precision and recall on test dataset\n",
    "    self.rev_label_dict = {self.label_dict[k]:k for k in self.label_dict.keys()} \n",
    "\n",
    "  def load_data(self, df):\n",
    "    MAX_LEN = 256 # dont need to enforce this now because snli is a sanitized dataset where sentence lenghts are reasonable. otherwise the beert model doesn't have enough parameters to handle long length sentences\n",
    "    token_ids = []\n",
    "    mask_ids = []\n",
    "    seg_ids = []\n",
    "    y = []\n",
    "\n",
    "    premise_list = df['unit1_txt'].to_list()\n",
    "    hypothesis_list = df['unit2_txt'].to_list()\n",
    "    label_list = df['label'].to_list()\n",
    "\n",
    "    for (premise, hypothesis, label) in zip(premise_list, hypothesis_list, label_list):\n",
    "      premise_id = self.tokenizer.encode(premise, add_special_tokens = False, max_length=MAX_LEN, truncation=True)\n",
    "      hypothesis_id = self.tokenizer.encode(hypothesis, add_special_tokens = False, max_length=MAX_LEN, truncation=True)\n",
    "      pair_token_ids = [self.tokenizer.cls_token_id] + premise_id + [self.tokenizer.sep_token_id] + hypothesis_id + [self.tokenizer.sep_token_id]\n",
    "      premise_len = len(premise_id)\n",
    "      hypothesis_len = len(hypothesis_id)\n",
    "\n",
    "      segment_ids = torch.tensor([0] * (premise_len + 2) + [1] * (hypothesis_len + 1))  # sentence 0 and sentence 1\n",
    "      attention_mask_ids = torch.tensor([1] * (premise_len + hypothesis_len + 3))  # mask padded values\n",
    "\n",
    "      token_ids.append(torch.tensor(pair_token_ids))\n",
    "      seg_ids.append(segment_ids)\n",
    "      mask_ids.append(attention_mask_ids)\n",
    "      y.append(self.label_dict[label])\n",
    "    \n",
    "    token_ids = pad_sequence(token_ids, batch_first=True)\n",
    "    mask_ids = pad_sequence(mask_ids, batch_first=True)\n",
    "    seg_ids = pad_sequence(seg_ids, batch_first=True)\n",
    "\n",
    "    y = torch.tensor(y)\n",
    "    dataset = TensorDataset(token_ids, mask_ids, seg_ids, y)\n",
    "    return dataset\n",
    "\n",
    "  def get_data_loaders(self, batch_size=32, shuffle=True):\n",
    "    train_loader = DataLoader(\n",
    "      self.train_data,\n",
    "      shuffle=shuffle,\n",
    "      batch_size=batch_size\n",
    "    )\n",
    "\n",
    "    val_loader = DataLoader(\n",
    "      self.val_data,\n",
    "      shuffle=shuffle,\n",
    "      batch_size=batch_size\n",
    "    )\n",
    "\n",
    "    test_loader = DataLoader(\n",
    "      self.test_data,\n",
    "      shuffle=shuffle,\n",
    "      batch_size=batch_size\n",
    "    )\n",
    "\n",
    "    return train_loader, val_loader, test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnli_dataset = MNLIDataBert(train_df, val_df, test_df)\n",
    "\n",
    "train_loader, val_loader, test_loader = mnli_dataset.get_data_loaders()\n",
    "label_dict = mnli_dataset.label_dict # required by custom func to calculate accuracy, bert model\n",
    "rev_label_dict = mnli_dataset.rev_label_dict # required by custom func to calculate accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "/home/VD/kaveri/anaconda3/envs/py310/lib/python3.10/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertForSequenceClassification, AdamW\n",
    "from torch import optim\n",
    "\n",
    "model = BertForSequenceClassification.from_pretrained(BERT_MODEL, num_labels=len(label_dict)).to(device)\n",
    "optimizer = AdamW(model.parameters(), lr=2e-5, correct_bias=False)\n",
    "scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.6, mode='max', patience=2, min_lr=5e-7, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# to evaluate model for train and test. And also use classification report for testing\n",
    "\n",
    "# helper function to calculate the batch accuracy\n",
    "def multi_acc(y_pred, y_test):\n",
    "  acc = (torch.log_softmax(y_pred, dim=1).argmax(dim=1) == y_test).sum().float() / float(y_test.size(0))\n",
    "  return acc\n",
    "\n",
    "# freeze model weights and measure validation / test \n",
    "def evaluate_accuracy(model, optimizer, data_loader, rev_label_dict):\n",
    "  model.eval()\n",
    "  total_val_acc  = 0\n",
    "  total_val_loss = 0\n",
    "  \n",
    "  #for classification report\n",
    "  y_true = []\n",
    "  y_pred = []\n",
    "  \n",
    "  with torch.no_grad():\n",
    "    for batch_idx, (pair_token_ids, mask_ids, seg_ids, y) in enumerate(data_loader):      \n",
    "      optimizer.zero_grad()\n",
    "      pair_token_ids = pair_token_ids.to(device)\n",
    "      mask_ids = mask_ids.to(device)\n",
    "      seg_ids = seg_ids.to(device)\n",
    "      labels = y.to(device)\n",
    "      \n",
    "      loss, prediction = model(pair_token_ids, \n",
    "                            token_type_ids=seg_ids, \n",
    "                            attention_mask=mask_ids, \n",
    "                            labels=labels).values()\n",
    "      \n",
    "      acc = multi_acc(prediction, labels)\n",
    "\n",
    "      total_val_loss += loss.item()\n",
    "      total_val_acc  += acc.item()\n",
    "\n",
    "      # log predictions for classification report\n",
    "      argmax_predictions = torch.argmax(prediction,dim=1).tolist()\n",
    "      labels_list = labels.tolist()\n",
    "      assert(len(labels_list)==len(argmax_predictions))\n",
    "      for p in argmax_predictions: y_pred.append(rev_label_dict[int(p)])\n",
    "      for l in labels_list: y_true.append(rev_label_dict[l])\n",
    "\n",
    "  val_acc  = total_val_acc/len(data_loader)\n",
    "  val_loss = total_val_loss/len(data_loader)\n",
    "  cr = classification_report(y_true, y_pred)\n",
    "  \n",
    "  return val_acc, val_loss, cr, model, optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ### HUGGINGFACE PLAYGROUND\n",
    "# from transformers import Trainer\n",
    "# class MyTrainer(Trainer):\n",
    "#   def compute_loss(self,model, inputs, rev_label_dict):\n",
    "#     pair_token_ids, seg_ids, mask_ids, labels = inputs\n",
    "#     loss = model(pair_token_ids, \n",
    "#                             token_type_ids=seg_ids, \n",
    "#                             attention_mask=mask_ids, \n",
    "#                             labels=labels)#.values()\n",
    "#     return loss #my_custom_loss(logits, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "### MODIFIED\n",
    "import time\n",
    "import traceback\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "def EarlyStoppingCallbackCustomBasedOnLoss(val_loss, prev_loss, trigger_times, patience, model):\n",
    "  # https://clay-atlas.com/us/blog/2021/08/25/pytorch-en-early-stopping/\n",
    "  if val_loss > prev_loss:\n",
    "    trigger_times += 1\n",
    "    print('trigger times:', trigger_times)\n",
    "    if trigger_times >= patience:\n",
    "        print('Early stopping!\\nStart to test process.')\n",
    "        return model\n",
    "  else:\n",
    "    print('trigger times: 0')\n",
    "    trigger_times = 0\n",
    "  prev_loss = val_loss\n",
    "  return prev_loss\n",
    "\n",
    "EPOCHS = 100\n",
    "\n",
    "def train(model, train_loader, val_loader, optimizer, scheduler, rev_label_dict):  \n",
    "  for epoch in range(EPOCHS):\n",
    "    start = time.time()\n",
    "    model.train()\n",
    "    total_train_loss = 0\n",
    "    total_train_acc  = 0\n",
    "\n",
    "    # logging for scheduler\n",
    "    losses = []\n",
    "    accuracies= []\n",
    "\n",
    "    # Early stopping\n",
    "    prev_loss = 100\n",
    "    patience = 12\n",
    "    trigger_times = 0\n",
    "\n",
    "    for batch_idx, (pair_token_ids, mask_ids, seg_ids, y) in enumerate(train_loader):\n",
    "      optimizer.zero_grad()\n",
    "      pair_token_ids = pair_token_ids.to(device)\n",
    "      mask_ids = mask_ids.to(device)\n",
    "      seg_ids = seg_ids.to(device)\n",
    "      labels = y.to(device)\n",
    "\n",
    "      try:\n",
    "        loss, prediction = model(pair_token_ids, \n",
    "                              token_type_ids=seg_ids, \n",
    "                              attention_mask=mask_ids, \n",
    "                              labels=labels).values()\n",
    "\n",
    "        acc = multi_acc(prediction, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        total_train_loss += loss.item()\n",
    "        total_train_acc  += acc.item()\n",
    "\n",
    "        # log losses for scheduler\n",
    "        losses.append(loss)\n",
    "        accuracies.append(acc)\n",
    "        mean_loss = sum(losses)/len(losses)\n",
    "        scheduler.step(mean_loss)\n",
    "\n",
    "       \n",
    "        \n",
    "      except Exception as e:\n",
    "        print(traceback.format_exc())\n",
    "        print('helpp')\n",
    "        break\n",
    "\n",
    "    train_acc  = total_train_acc/len(train_loader)\n",
    "    train_loss = total_train_loss/len(train_loader)\n",
    "\n",
    "    val_acc, val_loss, cr, model, optimizer = evaluate_accuracy(model, optimizer, val_loader, rev_label_dict)\n",
    "    prev_loss = EarlyStoppingCallbackCustomBasedOnLoss(val_loss, prev_loss, trigger_times, patience, model)\n",
    "\n",
    "    end = time.time()\n",
    "    hours, rem = divmod(end-start, 3600)\n",
    "    minutes, seconds = divmod(rem, 60)\n",
    "\n",
    "    print(f'Epoch {epoch+1}: train_loss: {train_loss:.4f} train_acc: {train_acc:.4f} | val_loss: {val_loss:.4f} val_acc: {val_acc:.4f}')\n",
    "    print(\"{:0>2}:{:0>2}:{:05.2f}\".format(int(hours),int(minutes),seconds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ### WORKING CODE\n",
    "# import time\n",
    "# import traceback\n",
    "\n",
    "# from sklearn.metrics import classification_report\n",
    "\n",
    "# EPOCHS = 1\n",
    "\n",
    "# def train(model, train_loader, val_loader, optimizer, rev_label_dict):  \n",
    "#   for epoch in range(EPOCHS):\n",
    "#     start = time.time()\n",
    "#     model.train()\n",
    "#     total_train_loss = 0\n",
    "#     total_train_acc  = 0\n",
    "#     for batch_idx, (pair_token_ids, mask_ids, seg_ids, y) in enumerate(train_loader):\n",
    "#       optimizer.zero_grad()\n",
    "#       pair_token_ids = pair_token_ids.to(device)\n",
    "#       mask_ids = mask_ids.to(device)\n",
    "#       seg_ids = seg_ids.to(device)\n",
    "#       labels = y.to(device)\n",
    "\n",
    "#       try:\n",
    "#         loss, prediction = model(pair_token_ids, \n",
    "#                               token_type_ids=seg_ids, \n",
    "#                               attention_mask=mask_ids, \n",
    "#                               labels=labels).values()\n",
    "\n",
    "#         acc = multi_acc(prediction, labels)\n",
    "#         loss.backward()\n",
    "#         optimizer.step()\n",
    "#         total_train_loss += loss.item()\n",
    "#         total_train_acc  += acc.item()\n",
    "#         # print(seg_ids.shape, pair_token_ids.shape, mask_ids.shape)\n",
    "#       except Exception as e:\n",
    "#         print(labels)\n",
    "#         print(seg_ids.shape, pair_token_ids.shape, mask_ids.shape)\n",
    "#         print(pair_token_ids)\n",
    "#         print(traceback.format_exc())\n",
    "#         print('helpp')\n",
    "#         break\n",
    "\n",
    "#     train_acc  = total_train_acc/len(train_loader)\n",
    "#     train_loss = total_train_loss/len(train_loader)\n",
    "\n",
    "#     val_acc, val_loss, cr, model, optimizer = evaluate_accuracy(model, optimizer, val_loader, rev_label_dict)\n",
    "#     end = time.time()\n",
    "#     hours, rem = divmod(end-start, 3600)\n",
    "#     minutes, seconds = divmod(rem, 60)\n",
    "\n",
    "#     print(f'Epoch {epoch+1}: train_loss: {train_loss:.4f} train_acc: {train_acc:.4f} | val_loss: {val_loss:.4f} val_acc: {val_acc:.4f}')\n",
    "#     print(\"{:0>2}:{:0>2}:{:05.2f}\".format(int(hours),int(minutes),seconds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00005: reducing learning rate of group 0 to 1.2000e-05.\n",
      "Epoch 00008: reducing learning rate of group 0 to 7.2000e-06.\n",
      "Epoch 00011: reducing learning rate of group 0 to 4.3200e-06.\n",
      "Epoch 00014: reducing learning rate of group 0 to 2.5920e-06.\n",
      "Epoch 00017: reducing learning rate of group 0 to 1.5552e-06.\n",
      "Epoch 00020: reducing learning rate of group 0 to 9.3312e-07.\n",
      "Epoch 00023: reducing learning rate of group 0 to 5.5987e-07.\n",
      "Epoch 00026: reducing learning rate of group 0 to 5.0000e-07.\n",
      "trigger times: 0\n",
      "Epoch 1: train_loss: 2.7123 train_acc: 0.2174 | val_loss: 2.7235 val_acc: 0.2109\n",
      "00:06:01.59\n",
      "trigger times: 0\n",
      "Epoch 2: train_loss: 2.6379 train_acc: 0.2295 | val_loss: 2.6636 val_acc: 0.2261\n",
      "00:06:01.36\n",
      "trigger times: 0\n",
      "Epoch 3: train_loss: 2.5749 train_acc: 0.2498 | val_loss: 2.5904 val_acc: 0.2597\n",
      "00:06:01.16\n",
      "trigger times: 0\n",
      "Epoch 4: train_loss: 2.4988 train_acc: 0.2846 | val_loss: 2.4989 val_acc: 0.3285\n",
      "00:06:01.16\n",
      "trigger times: 0\n",
      "Epoch 5: train_loss: 2.4177 train_acc: 0.3356 | val_loss: 2.4156 val_acc: 0.3483\n",
      "00:06:01.36\n",
      "trigger times: 0\n",
      "Epoch 6: train_loss: 2.3390 train_acc: 0.3633 | val_loss: 2.3418 val_acc: 0.3689\n",
      "00:06:01.21\n",
      "trigger times: 0\n",
      "Epoch 7: train_loss: 2.2710 train_acc: 0.3863 | val_loss: 2.2804 val_acc: 0.3848\n",
      "00:06:01.08\n",
      "trigger times: 0\n",
      "Epoch 8: train_loss: 2.2096 train_acc: 0.4025 | val_loss: 2.2245 val_acc: 0.4082\n",
      "00:06:01.52\n",
      "trigger times: 0\n",
      "Epoch 9: train_loss: 2.1530 train_acc: 0.4193 | val_loss: 2.1736 val_acc: 0.4098\n",
      "00:06:01.20\n",
      "trigger times: 0\n",
      "Epoch 10: train_loss: 2.0991 train_acc: 0.4328 | val_loss: 2.1322 val_acc: 0.4230\n",
      "00:06:01.20\n",
      "trigger times: 0\n",
      "Epoch 11: train_loss: 2.0500 train_acc: 0.4460 | val_loss: 2.0918 val_acc: 0.4299\n",
      "00:06:00.97\n",
      "trigger times: 0\n",
      "Epoch 12: train_loss: 2.0025 train_acc: 0.4614 | val_loss: 2.0612 val_acc: 0.4466\n",
      "00:06:01.11\n",
      "trigger times: 0\n",
      "Epoch 13: train_loss: 1.9583 train_acc: 0.4744 | val_loss: 2.0044 val_acc: 0.4600\n",
      "00:06:01.09\n",
      "trigger times: 0\n",
      "Epoch 14: train_loss: 1.9189 train_acc: 0.4826 | val_loss: 1.9947 val_acc: 0.4565\n",
      "00:06:01.08\n",
      "trigger times: 0\n",
      "Epoch 15: train_loss: 1.8816 train_acc: 0.4979 | val_loss: 1.9711 val_acc: 0.4622\n",
      "00:06:01.00\n",
      "trigger times: 0\n",
      "Epoch 16: train_loss: 1.8428 train_acc: 0.5076 | val_loss: 1.9393 val_acc: 0.4705\n",
      "00:06:00.90\n",
      "trigger times: 0\n",
      "Epoch 17: train_loss: 1.8086 train_acc: 0.5169 | val_loss: 1.9219 val_acc: 0.4726\n",
      "00:05:59.88\n",
      "trigger times: 0\n",
      "Epoch 18: train_loss: 1.7759 train_acc: 0.5264 | val_loss: 1.8955 val_acc: 0.4822\n",
      "00:06:01.08\n",
      "trigger times: 0\n",
      "Epoch 19: train_loss: 1.7447 train_acc: 0.5338 | val_loss: 1.8812 val_acc: 0.4813\n",
      "00:06:00.92\n",
      "trigger times: 0\n",
      "Epoch 20: train_loss: 1.7149 train_acc: 0.5400 | val_loss: 1.8616 val_acc: 0.4882\n",
      "00:06:00.12\n",
      "trigger times: 0\n",
      "Epoch 21: train_loss: 1.6853 train_acc: 0.5489 | val_loss: 1.8320 val_acc: 0.5017\n",
      "00:06:01.26\n",
      "trigger times: 0\n",
      "Epoch 22: train_loss: 1.6528 train_acc: 0.5593 | val_loss: 1.8229 val_acc: 0.5042\n",
      "00:06:01.20\n",
      "trigger times: 0\n",
      "Epoch 23: train_loss: 1.6304 train_acc: 0.5604 | val_loss: 1.8086 val_acc: 0.5063\n",
      "00:06:10.85\n",
      "trigger times: 0\n",
      "Epoch 24: train_loss: 1.6073 train_acc: 0.5680 | val_loss: 1.8033 val_acc: 0.5011\n",
      "00:06:00.87\n",
      "trigger times: 0\n",
      "Epoch 25: train_loss: 1.5784 train_acc: 0.5730 | val_loss: 1.7789 val_acc: 0.5164\n",
      "00:06:01.00\n",
      "trigger times: 0\n",
      "Epoch 26: train_loss: 1.5593 train_acc: 0.5808 | val_loss: 1.7684 val_acc: 0.5105\n",
      "00:05:59.94\n",
      "trigger times: 0\n",
      "Epoch 27: train_loss: 1.5340 train_acc: 0.5855 | val_loss: 1.7786 val_acc: 0.5089\n",
      "00:06:01.13\n",
      "trigger times: 0\n",
      "Epoch 28: train_loss: 1.5122 train_acc: 0.5908 | val_loss: 1.7526 val_acc: 0.5112\n",
      "00:06:01.15\n",
      "trigger times: 0\n",
      "Epoch 29: train_loss: 1.4910 train_acc: 0.5945 | val_loss: 1.7433 val_acc: 0.5146\n",
      "00:06:01.09\n",
      "trigger times: 0\n",
      "Epoch 30: train_loss: 1.4696 train_acc: 0.6022 | val_loss: 1.7313 val_acc: 0.5121\n",
      "00:06:01.18\n",
      "trigger times: 0\n",
      "Epoch 31: train_loss: 1.4513 train_acc: 0.6077 | val_loss: 1.7165 val_acc: 0.5194\n",
      "00:06:01.13\n",
      "trigger times: 0\n",
      "Epoch 32: train_loss: 1.4328 train_acc: 0.6124 | val_loss: 1.7190 val_acc: 0.5178\n",
      "00:06:01.11\n",
      "trigger times: 0\n",
      "Epoch 33: train_loss: 1.4126 train_acc: 0.6159 | val_loss: 1.7084 val_acc: 0.5201\n",
      "00:03:05.98\n",
      "trigger times: 0\n",
      "Epoch 34: train_loss: 1.3927 train_acc: 0.6180 | val_loss: 1.6975 val_acc: 0.5260\n",
      "00:02:39.62\n",
      "trigger times: 0\n",
      "Epoch 35: train_loss: 1.3740 train_acc: 0.6259 | val_loss: 1.6977 val_acc: 0.5241\n",
      "00:02:39.35\n",
      "trigger times: 0\n",
      "Epoch 36: train_loss: 1.3604 train_acc: 0.6278 | val_loss: 1.6942 val_acc: 0.5261\n",
      "00:02:39.35\n",
      "trigger times: 0\n",
      "Epoch 37: train_loss: 1.3404 train_acc: 0.6348 | val_loss: 1.6854 val_acc: 0.5233\n",
      "00:02:39.10\n",
      "trigger times: 0\n",
      "Epoch 38: train_loss: 1.3262 train_acc: 0.6399 | val_loss: 1.7011 val_acc: 0.5287\n",
      "00:02:39.37\n",
      "trigger times: 0\n",
      "Epoch 39: train_loss: 1.3099 train_acc: 0.6439 | val_loss: 1.6668 val_acc: 0.5357\n",
      "00:02:39.21\n",
      "trigger times: 0\n",
      "Epoch 40: train_loss: 1.2948 train_acc: 0.6460 | val_loss: 1.6763 val_acc: 0.5316\n",
      "00:02:39.39\n",
      "trigger times: 0\n",
      "Epoch 41: train_loss: 1.2784 train_acc: 0.6538 | val_loss: 1.6793 val_acc: 0.5330\n",
      "00:02:39.33\n",
      "trigger times: 0\n",
      "Epoch 42: train_loss: 1.2603 train_acc: 0.6563 | val_loss: 1.6656 val_acc: 0.5407\n",
      "00:02:39.65\n",
      "trigger times: 0\n",
      "Epoch 43: train_loss: 1.2453 train_acc: 0.6596 | val_loss: 1.6663 val_acc: 0.5373\n",
      "00:02:39.29\n",
      "trigger times: 0\n",
      "Epoch 44: train_loss: 1.2318 train_acc: 0.6645 | val_loss: 1.6558 val_acc: 0.5373\n",
      "00:02:39.41\n",
      "trigger times: 0\n",
      "Epoch 45: train_loss: 1.2192 train_acc: 0.6678 | val_loss: 1.6582 val_acc: 0.5392\n",
      "00:02:39.42\n",
      "trigger times: 0\n",
      "Epoch 46: train_loss: 1.2037 train_acc: 0.6707 | val_loss: 1.6429 val_acc: 0.5473\n",
      "00:02:40.17\n",
      "trigger times: 0\n",
      "Epoch 47: train_loss: 1.1928 train_acc: 0.6768 | val_loss: 1.6596 val_acc: 0.5438\n",
      "00:02:39.49\n",
      "trigger times: 0\n",
      "Epoch 48: train_loss: 1.1755 train_acc: 0.6789 | val_loss: 1.6557 val_acc: 0.5415\n",
      "00:02:39.40\n",
      "trigger times: 0\n",
      "Epoch 49: train_loss: 1.1618 train_acc: 0.6838 | val_loss: 1.6397 val_acc: 0.5430\n",
      "00:02:39.52\n",
      "trigger times: 0\n",
      "Epoch 50: train_loss: 1.1501 train_acc: 0.6872 | val_loss: 1.6624 val_acc: 0.5347\n",
      "00:02:39.56\n",
      "trigger times: 0\n",
      "Epoch 51: train_loss: 1.1337 train_acc: 0.6891 | val_loss: 1.6673 val_acc: 0.5347\n",
      "00:02:39.71\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/home/VD/kaveri/bert_categorical_tutorial/analytics_vidhya_snli_medium copy.ipynb Cell 18'\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell://attached-container%2B7b22636f6e7461696e65724e616d65223a222f636f6c695f6b61766572695f74657374222c2273657474696e6773223a7b22686f7374223a227373683a2f2f6d795f746f6e795f736572766572227d7d/home/VD/kaveri/bert_categorical_tutorial/analytics_vidhya_snli_medium%20copy.ipynb#ch0000017vscode-remote?line=2'>3</a>\u001b[0m \u001b[39mwith\u001b[39;00m warnings\u001b[39m.\u001b[39mcatch_warnings():\n\u001b[1;32m      <a href='vscode-notebook-cell://attached-container%2B7b22636f6e7461696e65724e616d65223a222f636f6c695f6b61766572695f74657374222c2273657474696e6773223a7b22686f7374223a227373683a2f2f6d795f746f6e795f736572766572227d7d/home/VD/kaveri/bert_categorical_tutorial/analytics_vidhya_snli_medium%20copy.ipynb#ch0000017vscode-remote?line=3'>4</a>\u001b[0m     warnings\u001b[39m.\u001b[39msimplefilter(\u001b[39m\"\u001b[39m\u001b[39mignore\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m----> <a href='vscode-notebook-cell://attached-container%2B7b22636f6e7461696e65724e616d65223a222f636f6c695f6b61766572695f74657374222c2273657474696e6773223a7b22686f7374223a227373683a2f2f6d795f746f6e795f736572766572227d7d/home/VD/kaveri/bert_categorical_tutorial/analytics_vidhya_snli_medium%20copy.ipynb#ch0000017vscode-remote?line=5'>6</a>\u001b[0m     train(model, train_loader, val_loader, optimizer, scheduler, rev_label_dict)\n",
      "\u001b[1;32m/home/VD/kaveri/bert_categorical_tutorial/analytics_vidhya_snli_medium copy.ipynb Cell 16'\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(model, train_loader, val_loader, optimizer, scheduler, rev_label_dict)\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell://attached-container%2B7b22636f6e7461696e65724e616d65223a222f636f6c695f6b61766572695f74657374222c2273657474696e6773223a7b22686f7374223a227373683a2f2f6d795f746f6e795f736572766572227d7d/home/VD/kaveri/bert_categorical_tutorial/analytics_vidhya_snli_medium%20copy.ipynb#ch0000015vscode-remote?line=46'>47</a>\u001b[0m loss, prediction \u001b[39m=\u001b[39m model(pair_token_ids, \n\u001b[1;32m     <a href='vscode-notebook-cell://attached-container%2B7b22636f6e7461696e65724e616d65223a222f636f6c695f6b61766572695f74657374222c2273657474696e6773223a7b22686f7374223a227373683a2f2f6d795f746f6e795f736572766572227d7d/home/VD/kaveri/bert_categorical_tutorial/analytics_vidhya_snli_medium%20copy.ipynb#ch0000015vscode-remote?line=47'>48</a>\u001b[0m                       token_type_ids\u001b[39m=\u001b[39mseg_ids, \n\u001b[1;32m     <a href='vscode-notebook-cell://attached-container%2B7b22636f6e7461696e65724e616d65223a222f636f6c695f6b61766572695f74657374222c2273657474696e6773223a7b22686f7374223a227373683a2f2f6d795f746f6e795f736572766572227d7d/home/VD/kaveri/bert_categorical_tutorial/analytics_vidhya_snli_medium%20copy.ipynb#ch0000015vscode-remote?line=48'>49</a>\u001b[0m                       attention_mask\u001b[39m=\u001b[39mmask_ids, \n\u001b[1;32m     <a href='vscode-notebook-cell://attached-container%2B7b22636f6e7461696e65724e616d65223a222f636f6c695f6b61766572695f74657374222c2273657474696e6773223a7b22686f7374223a227373683a2f2f6d795f746f6e795f736572766572227d7d/home/VD/kaveri/bert_categorical_tutorial/analytics_vidhya_snli_medium%20copy.ipynb#ch0000015vscode-remote?line=49'>50</a>\u001b[0m                       labels\u001b[39m=\u001b[39mlabels)\u001b[39m.\u001b[39mvalues()\n\u001b[1;32m     <a href='vscode-notebook-cell://attached-container%2B7b22636f6e7461696e65724e616d65223a222f636f6c695f6b61766572695f74657374222c2273657474696e6773223a7b22686f7374223a227373683a2f2f6d795f746f6e795f736572766572227d7d/home/VD/kaveri/bert_categorical_tutorial/analytics_vidhya_snli_medium%20copy.ipynb#ch0000015vscode-remote?line=51'>52</a>\u001b[0m acc \u001b[39m=\u001b[39m multi_acc(prediction, labels)\n\u001b[0;32m---> <a href='vscode-notebook-cell://attached-container%2B7b22636f6e7461696e65724e616d65223a222f636f6c695f6b61766572695f74657374222c2273657474696e6773223a7b22686f7374223a227373683a2f2f6d795f746f6e795f736572766572227d7d/home/VD/kaveri/bert_categorical_tutorial/analytics_vidhya_snli_medium%20copy.ipynb#ch0000015vscode-remote?line=52'>53</a>\u001b[0m loss\u001b[39m.\u001b[39;49mbackward()\n\u001b[1;32m     <a href='vscode-notebook-cell://attached-container%2B7b22636f6e7461696e65724e616d65223a222f636f6c695f6b61766572695f74657374222c2273657474696e6773223a7b22686f7374223a227373683a2f2f6d795f746f6e795f736572766572227d7d/home/VD/kaveri/bert_categorical_tutorial/analytics_vidhya_snli_medium%20copy.ipynb#ch0000015vscode-remote?line=53'>54</a>\u001b[0m optimizer\u001b[39m.\u001b[39mstep()\n\u001b[1;32m     <a href='vscode-notebook-cell://attached-container%2B7b22636f6e7461696e65724e616d65223a222f636f6c695f6b61766572695f74657374222c2273657474696e6773223a7b22686f7374223a227373683a2f2f6d795f746f6e795f736572766572227d7d/home/VD/kaveri/bert_categorical_tutorial/analytics_vidhya_snli_medium%20copy.ipynb#ch0000015vscode-remote?line=54'>55</a>\u001b[0m total_train_loss \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m loss\u001b[39m.\u001b[39mitem()\n",
      "File \u001b[0;32m/home/VD/kaveri/anaconda3/envs/py310/lib/python3.10/site-packages/torch/_tensor.py:363\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    354\u001b[0m \u001b[39mif\u001b[39;00m has_torch_function_unary(\u001b[39mself\u001b[39m):\n\u001b[1;32m    355\u001b[0m     \u001b[39mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m    356\u001b[0m         Tensor\u001b[39m.\u001b[39mbackward,\n\u001b[1;32m    357\u001b[0m         (\u001b[39mself\u001b[39m,),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    361\u001b[0m         create_graph\u001b[39m=\u001b[39mcreate_graph,\n\u001b[1;32m    362\u001b[0m         inputs\u001b[39m=\u001b[39minputs)\n\u001b[0;32m--> 363\u001b[0m torch\u001b[39m.\u001b[39;49mautograd\u001b[39m.\u001b[39;49mbackward(\u001b[39mself\u001b[39;49m, gradient, retain_graph, create_graph, inputs\u001b[39m=\u001b[39;49minputs)\n",
      "File \u001b[0;32m/home/VD/kaveri/anaconda3/envs/py310/lib/python3.10/site-packages/torch/autograd/__init__.py:173\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    168\u001b[0m     retain_graph \u001b[39m=\u001b[39m create_graph\n\u001b[1;32m    170\u001b[0m \u001b[39m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[1;32m    171\u001b[0m \u001b[39m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m    172\u001b[0m \u001b[39m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 173\u001b[0m Variable\u001b[39m.\u001b[39;49m_execution_engine\u001b[39m.\u001b[39;49mrun_backward(  \u001b[39m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m    174\u001b[0m     tensors, grad_tensors_, retain_graph, create_graph, inputs,\n\u001b[1;32m    175\u001b[0m     allow_unreachable\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, accumulate_grad\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "from sklearn.exceptions import DataConversionWarning\n",
    "with warnings.catch_warnings():\n",
    "    warnings.simplefilter(\"ignore\")\n",
    "\n",
    "    train(model, train_loader, val_loader, optimizer, scheduler, rev_label_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(model.state_dict(), 'bert-nli.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test_loss: 1.5502 test_acc: 0.5702\n",
      "00:00:03.77\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "  antithesis       0.67      0.05      0.10        38\n",
      " attribution       0.76      0.87      0.81       101\n",
      "  background       0.21      0.15      0.17        96\n",
      "       cause       0.42      0.55      0.48        38\n",
      "circumstance       0.56      0.77      0.65        74\n",
      "  concession       0.32      0.53      0.40        51\n",
      "   condition       0.66      0.85      0.74        47\n",
      "    contrast       0.38      0.23      0.29        56\n",
      " elaboration       0.65      0.72      0.68       531\n",
      "  evaluation       0.38      0.43      0.40       115\n",
      "    evidence       0.73      0.42      0.53        76\n",
      "       joint       0.54      0.63      0.58       346\n",
      "     justify       0.00      0.00      0.00        49\n",
      "      manner       1.00      0.10      0.17        21\n",
      "       means       0.87      0.76      0.81        17\n",
      "  motivation       0.00      0.00      0.00        14\n",
      " preparation       0.60      0.67      0.64       116\n",
      "     purpose       0.78      0.85      0.81        66\n",
      "    question       0.57      0.84      0.68        31\n",
      " restatement       0.52      0.29      0.37        45\n",
      "      result       0.50      0.16      0.24        37\n",
      "    sequence       0.52      0.43      0.47       124\n",
      "solutionhood       0.00      0.00      0.00         2\n",
      "\n",
      "    accuracy                           0.57      2091\n",
      "   macro avg       0.51      0.45      0.44      2091\n",
      "weighted avg       0.55      0.57      0.54      2091\n",
      "\n",
      "Test Loss: 1.550 |  Test Acc: 57.02%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/VD/kaveri/anaconda3/envs/py310/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1327: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/VD/kaveri/anaconda3/envs/py310/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1327: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/VD/kaveri/anaconda3/envs/py310/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1327: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "def validate(model, test_loader, optimizer, rev_label_dict):\n",
    "  start = time.time()\n",
    "  test_acc, test_loss, cr, model, optimizer = evaluate_accuracy(model, optimizer, test_loader, rev_label_dict)\n",
    "  end = time.time()\n",
    "  hours, rem = divmod(end-start, 3600)\n",
    "  minutes, seconds = divmod(rem, 60)\n",
    "\n",
    "  print(f'Test_loss: {test_loss:.4f} test_acc: {test_acc:.4f}')\n",
    "  print(\"{:0>2}:{:0>2}:{:05.2f}\".format(int(hours),int(minutes),seconds))\n",
    "  print(cr)\n",
    "\n",
    "  return test_loss, test_acc\n",
    "\n",
    "\n",
    "# model.load_state_dict(torch.load('bert-nli.pt'))\n",
    "test_loss, test_acc = validate(model, test_loader, optimizer, rev_label_dict)\n",
    "print(f'Test Loss: {test_loss:.3f} |  Test Acc: {test_acc*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: Another wrapper to return basic predictions/log them in it as well."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "e3409ea685db85227fbd9509d1b1ace14d085473eb2d57f3ba9dd0302d25f838"
  },
  "kernelspec": {
   "display_name": "Python 3.10.4 ('py310')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}