rails_experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 1\n",
    "%aimport rails,aise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from collections import deque\n",
    "import numpy as np\n",
    "from torchvision import datasets, transforms\n",
    "from torch.utils.data import DataLoader\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import time\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import rails\n",
    "import model as _model\n",
    "import utils as _tools\n",
    "\n",
    "from resnet import ResNet18\n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import auc\n",
    "from sklearn.metrics import precision_recall_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set the deive\n",
    "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done within 28.223 secs.\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "initialize the trainned model\n",
    "\"\"\"\n",
    "\n",
    "start = time.perf_counter()\n",
    "\n",
    "resnet = _model.TransferModel(use_cpu=False)\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    resnet.model.load_state_dict(torch.load(\"models/Cardiomegaly_resnet18.pth\"))\n",
    "else:\n",
    "    resnet.model.load_state_dict(torch.load(\"models/Cardiomegaly_resnet18.pth\", map_location=torch.device('cpu')))\n",
    "resnet.model.to(DEVICE)\n",
    "resnet.model.eval()\n",
    "\n",
    "end = time.perf_counter()\n",
    "\n",
    "print(f\"Done within {end-start:.3f} secs.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "import resenet architechure\n",
    "\"\"\"\n",
    "\n",
    "model = ResNet18(num_classes=2)\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    model.load_state_dict(torch.load(\"./models/chexpert_resnet18.pt\"))\n",
    "else:    \n",
    "    model.load_state_dict(torch.load(\"./models/chexpert_resnet18.pt\", map_location=torch.device('cpu')))\n",
    "model.to(DEVICE)\n",
    "model.eval()\n",
    "\n",
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reformatter(loader, n_rows, perturb_std):\n",
    "    \"\"\"A function to reformat data from ther dataloaders\n",
    "    \n",
    "    args:\n",
    "       : loader (dataloader)\n",
    "       : n_rows (int): number of rows to sample\n",
    "       : perturb_std (float or None): the standard deviation of the \n",
    "           perturbations\n",
    "        \n",
    "    returns:\n",
    "        : x (torch.FloatTensor): the x data\n",
    "        : x_pert (torch.FloatTensor): the perturbed x data\n",
    "        : y (torch.LongTensor): the y data\n",
    "    \"\"\"\n",
    "    row_count = 0\n",
    "    \n",
    "    x = []\n",
    "    y = []\n",
    "    x_pert = []\n",
    "    \n",
    "    for image, labels in loader:\n",
    "        row_count += 1\n",
    "        x.append(image)\n",
    "        \n",
    "        if not perturb_std is None:\n",
    "            noise = np.random.normal(loc=0.0, \n",
    "                                  scale=perturb_std, \n",
    "                                  size=image.shape)\n",
    "            x_pert.append(noise + image)\n",
    "\n",
    "        y.append(labels[0]) # the label for cadiomegaly\n",
    "\n",
    "        if row_count == n_rows:\n",
    "            break\n",
    "            \n",
    "    x = torch.FloatTensor(x)\n",
    "    x_pert = torch.FloatTensor(x_pert)\n",
    "    y = torch.LongTensor(y)\n",
    "    \n",
    "    return x, x_pert, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train: torch.Size([37500, 1, 224, 224])\n",
      "x_train_p: torch.Size([0])\n",
      "y_train: torch.Size([37500])\n",
      "done\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-6-17182fe25508>:38: DeprecationWarning: an integer is required (got type numpy.float32).  Implicit conversion to integers using __int__ is deprecated, and may be removed in a future version of Python.\n",
      "  y = torch.LongTensor(y)\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "reformat the trainning data\n",
    "\"\"\" \n",
    "\n",
    "PERTURBATION_AMT_STD = None\n",
    "LOADER = resnet.dataloader_train.dataset\n",
    "TRAIN_ROWS = 100000\n",
    "\n",
    "x_train, x_train_p, y_train = reformatter(LOADER, TRAIN_ROWS, PERTURBATION_AMT_STD)\n",
    "\n",
    "print(f\"x_train: {x_train.shape}\")\n",
    "print(f\"x_train_p: {x_train_p.shape}\")\n",
    "print(f\"y_train: {y_train.shape}\")\n",
    "\n",
    "\n",
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_test: torch.Size([234, 1, 224, 224])\n",
      "x_test_p: torch.Size([234, 1, 224, 224])\n",
      "y_test: torch.Size([234])\n",
      "done\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-6-17182fe25508>:38: DeprecationWarning: an integer is required (got type numpy.float32).  Implicit conversion to integers using __int__ is deprecated, and may be removed in a future version of Python.\n",
      "  y = torch.LongTensor(y)\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "validation data reformatting\n",
    "\"\"\"\n",
    "\n",
    "PERTURBATION_AMT_STD = 0.01\n",
    "LOADER = resnet.dataloader_valid.dataset\n",
    "TEST_ROWS = 300 # just needs to be more than 234\n",
    "\n",
    "x_test, x_test_p, y_test = reformatter(LOADER, TEST_ROWS, PERTURBATION_AMT_STD)\n",
    "\n",
    "print(f\"x_test: {x_test.shape}\")\n",
    "print(f\"x_test_p: {x_test_p.shape}\")\n",
    "print(f\"y_test: {y_test.shape}\")\n",
    "\n",
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>optimal_threshold</th>\n",
       "      <td>0.337538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>true negatives</th>\n",
       "      <td>109.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>true positives</th>\n",
       "      <td>51.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>false positives</th>\n",
       "      <td>57.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>false negatives</th>\n",
       "      <td>17.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sensitivity</th>\n",
       "      <td>0.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specificity</th>\n",
       "      <td>0.656627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1-score</th>\n",
       "      <td>0.579545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.472222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AUCROC</th>\n",
       "      <td>0.756821</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            0\n",
       "optimal_threshold    0.337538\n",
       "true negatives     109.000000\n",
       "true positives      51.000000\n",
       "false positives     57.000000\n",
       "false negatives     17.000000\n",
       "sensitivity          0.750000\n",
       "specificity          0.656627\n",
       "F1-score             0.579545\n",
       "precision            0.472222\n",
       "recall               0.750000\n",
       "AUCROC               0.756821"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "evaluate ResNet18\n",
    "\"\"\"\n",
    "\n",
    "# get results on dev set\n",
    "results = resnet.evaluate_model(resnet.model.state_dict(), \n",
    "                                resnet.dataloader_valid, \n",
    "                                resnet.valid_map)\n",
    "\n",
    "res = _tools.get_classification_metrics(results)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done within 298.690 secs.\n"
     ]
    }
   ],
   "source": [
    "start = time.perf_counter()\n",
    "\n",
    "CONFIG = {\n",
    "    \"start_layer\": 1,\n",
    "    \"n_class\": 2,\n",
    "    \"aise_params\": [\n",
    "        {\"hidden_layer\": 3, \n",
    "         \"sampling_temperature\": 1, \n",
    "         \"max_generation\": 2, \n",
    "         \"n_neighbors\" : 5,\n",
    "         \"n_population\" : 2 * 5,\n",
    "         \"mut_range\": (.005, .015)}, \n",
    "    ]\n",
    "}\n",
    "\n",
    "rails_clf = rails.RAILS(model, \n",
    "              CONFIG, \n",
    "              x_train,\n",
    "              y_train)\n",
    "\n",
    "\n",
    "y_proba = rails_clf.predict(x_test)\n",
    "y_pred = y_proba.argmax(axis=1)\n",
    "\n",
    "end = time.perf_counter()\n",
    "print(f\"Done within {end-start:.3f} secs.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_metrics(results):\n",
    "    \"\"\"A function to return classification metrics \n",
    "    \n",
    "    args:\n",
    "        : results (pd.DataFrame): with columns [y_true, y_pred, y_prob]\n",
    "        \n",
    "    returns:\n",
    "        : metrics (pd.DataFrame): results\n",
    "    \"\"\"\n",
    "    metrics = {}\n",
    "    \n",
    "    y_prob = results['y_prob'].astype(float)\n",
    "    y_true = results['y_true'].astype(int)\n",
    "    y_pred = results['y_pred'].astype(int)\n",
    "    \n",
    "    # confusion matrix\n",
    "    cm = confusion_matrix(y_true, y_pred)\n",
    "    tn, fp, fn, tp = cm.ravel()\n",
    "    \n",
    "    metrics['true_positive'] = int(tp)\n",
    "    metrics['true_negative'] = int(tn)\n",
    "    metrics['false_positive'] = int(fp)\n",
    "    metrics['false_negative'] = int(fn)\n",
    "    \n",
    "    metrics['accuracy'] = (tp + tn) / (tn + tp + fn + fp)\n",
    "    metrics['precision'] = tp / (tp + fp)\n",
    "    metrics['recall'] = tp / (tp + fn)\n",
    "    metrics['f1_score'] = 2 * tp / (2*tp + fp + fn)\n",
    "    \n",
    "    metrics['aucroc'] = roc_auc_score(y_true, y_prob)\n",
    "\n",
    "    precision, recall, _ = precision_recall_curve(y_true, y_prob)\n",
    "    metrics['aucpr'] = auc(recall, precision)\n",
    "    \n",
    "    metrics = pd.DataFrame.from_dict(metrics, orient='index').round(4)\n",
    "    return metrics    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>true_positive</th>\n",
       "      <td>19.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>true_negative</th>\n",
       "      <td>144.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>false_positive</th>\n",
       "      <td>22.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>false_negative</th>\n",
       "      <td>49.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.6966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.4634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.2794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1_score</th>\n",
       "      <td>0.3486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aucroc</th>\n",
       "      <td>0.5734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aucpr</th>\n",
       "      <td>0.4761</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       0\n",
       "true_positive    19.0000\n",
       "true_negative   144.0000\n",
       "false_positive   22.0000\n",
       "false_negative   49.0000\n",
       "accuracy          0.6966\n",
       "precision         0.4634\n",
       "recall            0.2794\n",
       "f1_score          0.3486\n",
       "aucroc            0.5734\n",
       "aucpr             0.4761"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "get classification metrics\n",
    "\"\"\"\n",
    "\n",
    "df = pd.DataFrame({\n",
    "    \"y_true\": y_test.numpy(),\n",
    "    \"y_pred\": y_pred,\n",
    "    \"y_prob\": y_proba[:,1]\n",
    "})\n",
    "\n",
    "get_metrics(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done within 341.668 secs.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>true_positive</th>\n",
       "      <td>16.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>true_negative</th>\n",
       "      <td>150.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>false_positive</th>\n",
       "      <td>16.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>false_negative</th>\n",
       "      <td>52.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.7094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.2353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1_score</th>\n",
       "      <td>0.3200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aucroc</th>\n",
       "      <td>0.5695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aucpr</th>\n",
       "      <td>0.4788</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       0\n",
       "true_positive    16.0000\n",
       "true_negative   150.0000\n",
       "false_positive   16.0000\n",
       "false_negative   52.0000\n",
       "accuracy          0.7094\n",
       "precision         0.5000\n",
       "recall            0.2353\n",
       "f1_score          0.3200\n",
       "aucroc            0.5695\n",
       "aucpr             0.4788"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "test on the perturbed data\n",
    "\"\"\"\n",
    "\n",
    "y_proba = rails_clf.predict(x_test_p)\n",
    "y_pred = y_proba.argmax(axis=1)\n",
    "\n",
    "end = time.perf_counter()\n",
    "print(f\"Done within {end-start:.3f} secs.\")\n",
    "\n",
    "\"\"\"\n",
    "get classification metrics\n",
    "\"\"\"\n",
    "\n",
    "df = pd.DataFrame({\n",
    "    \"y_true\": y_test.numpy(),\n",
    "    \"y_pred\": y_pred,\n",
    "    \"y_prob\": y_proba[:,1]\n",
    "})\n",
    "\n",
    "df.head()\n",
    "\n",
    "get_metrics(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "class Dataset(torch.utils.data.Dataset):\n",
    "    \"\"\" Characterizes a dataset for PyTorch \"\"\"\n",
    "    def __init__(self, data, labels):\n",
    "        'Initialization'\n",
    "        self.labels = labels\n",
    "        self.data = data\n",
    "\n",
    "    def __len__(self):\n",
    "        'Denotes the total number of samples'\n",
    "        return len(self.data)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        'Generates one sample of data'\n",
    "        \n",
    "        image = self.data[index]\n",
    "        label = self.labels[index]\n",
    "        \n",
    "        return (image, label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>true_positive</th>\n",
       "      <td>7.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>true_negative</th>\n",
       "      <td>162.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>false_positive</th>\n",
       "      <td>4.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>false_negative</th>\n",
       "      <td>61.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.7222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.6364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.1029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1_score</th>\n",
       "      <td>0.1772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aucroc</th>\n",
       "      <td>0.7640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aucpr</th>\n",
       "      <td>0.5762</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       0\n",
       "true_positive     7.0000\n",
       "true_negative   162.0000\n",
       "false_positive    4.0000\n",
       "false_negative   61.0000\n",
       "accuracy          0.7222\n",
       "precision         0.6364\n",
       "recall            0.1029\n",
       "f1_score          0.1772\n",
       "aucroc            0.7640\n",
       "aucpr             0.5762"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "evaluate ResNet on perturbed images \n",
    "\"\"\"\n",
    "\n",
    "new_rows = []\n",
    "\n",
    "t = Dataset(x_test_p, y_test)\n",
    "x_perturbed = DataLoader(t, batch_size=1, shuffle=False)\n",
    "label_map = {resnet.condition : 0}\n",
    "\n",
    "for i, (inputs, label) in enumerate(x_perturbed):\n",
    "    output = resnet.model(inputs.to(resnet.device)).to(resnet.device)\n",
    "\n",
    "    _, y_pred = torch.max(output, 1)\n",
    "    y_prob = torch.sigmoid(output)\n",
    "    top_p, _ = y_prob.topk(1, dim=1)\n",
    "    \n",
    "    row = {\n",
    "        'y_prob': 1 - top_p.cpu().detach().numpy()[0][0],\n",
    "        'y_pred': y_pred.cpu().detach().numpy()[0],\n",
    "        'y_true': label.cpu().detach().numpy()[0]\n",
    "    }\n",
    "    \n",
    "    new_rows.append(row)\n",
    "\n",
    "\n",
    "results2 = pd.DataFrame(new_rows)       \n",
    "\n",
    "get_metrics(results2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}