pytorch_dqn_space_invaders.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports and Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/claytyler/cs6476/miniconda3/lib/python3.8/site-packages/gym/envs/registration.py:14: PkgResourcesDeprecationWarning: Parameters to load are deprecated.  Call .resolve and .require separately.\n",
      "  result = entry_point.load(False)\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import math\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import namedtuple\n",
    "from itertools import count\n",
    "from PIL import Image\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import torchvision.transforms as T\n",
    "\n",
    "\n",
    "env = gym.make('CartPole-v0').unwrapped\n",
    "\n",
    "# set up matplotlib\n",
    "is_ipython = 'inline' in matplotlib.get_backend()\n",
    "if is_ipython:\n",
    "    from IPython import display\n",
    "\n",
    "plt.ion()\n",
    "\n",
    "# if gpu is to be used\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Replay Buffer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "Transition = namedtuple('Transition',\n",
    "                        ('state', 'action', 'next_state', 'reward'))\n",
    "\n",
    "\n",
    "class ReplayMemory(object):\n",
    "\n",
    "    def __init__(self, capacity):\n",
    "        self.capacity = capacity\n",
    "        self.memory = []\n",
    "        self.position = 0\n",
    "\n",
    "    def push(self, *args):\n",
    "        \"\"\"Saves a transition.\"\"\"\n",
    "        if len(self.memory) < self.capacity:\n",
    "            self.memory.append(None)\n",
    "        self.memory[self.position] = Transition(*args)\n",
    "        self.position = (self.position + 1) % self.capacity\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        return random.sample(self.memory, batch_size)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.memory)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Architectures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN(nn.Module):\n",
    "\n",
    "    def __init__(self, h, w, outputs):\n",
    "        super(DQN, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 16, kernel_size=5, stride=2)\n",
    "        self.bn1 = nn.BatchNorm2d(16)\n",
    "        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, stride=2)\n",
    "        self.bn2 = nn.BatchNorm2d(32)\n",
    "        self.conv3 = nn.Conv2d(32, 32, kernel_size=5, stride=2)\n",
    "        self.bn3 = nn.BatchNorm2d(32)\n",
    "\n",
    "        # Number of Linear input connections depends on output of conv2d layers\n",
    "        # and therefore the input image size, so compute it.\n",
    "        def conv2d_size_out(size, kernel_size = 5, stride = 2):\n",
    "            return (size - (kernel_size - 1) - 1) // stride  + 1\n",
    "        convw = conv2d_size_out(conv2d_size_out(conv2d_size_out(w)))\n",
    "        convh = conv2d_size_out(conv2d_size_out(conv2d_size_out(h)))\n",
    "        linear_input_size = convw * convh * 32\n",
    "        self.head = nn.Linear(linear_input_size, outputs)\n",
    "\n",
    "    # Called with either one element to determine next action, or a batch\n",
    "    # during optimization. Returns tensor([[left0exp,right0exp]...]).\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.bn1(self.conv1(x)))\n",
    "        x = F.relu(self.bn2(self.conv2(x)))\n",
    "        x = F.relu(self.bn3(self.conv3(x)))\n",
    "        return self.head(x.view(x.size(0), -1))\n",
    "    \n",
    "class ConvQNet(nn.Module):\n",
    "    def __init__(self, env, config, logger=None):\n",
    "        super().__init__()\n",
    "\n",
    "        H,W,C = env.observation_space.shape\n",
    "        no_actions = env.action_space.n\n",
    "        stacked_obs = config.state_history\n",
    "\n",
    "        self.layer1 = nn.Conv2d(C*stacked_obs, 16, stride = 4, kernel_size = 8) #16 8x8\n",
    "        self.layer2 = nn.ReLU()\n",
    "        self.layer3 = nn.Conv2d(16, 32, stride = 2, kernel_size = 4) #32 4x4\n",
    "        self.layer4 = nn.ReLU()\n",
    "        s1 = int((H-8)/4) + 1 #output of layer1\n",
    "        s2 = int((s1-4)/2) + 1 #output of layer3\n",
    "        in_size = 32*s2*s2\n",
    "        self.layer5 = nn.Linear(in_size, no_actions)\n",
    "\n",
    "    def forward(self, state):\n",
    "        state = state.permute(0,3,2,1)\n",
    "        a0 = state.shape[0]\n",
    "        state = self.layer1(state)\n",
    "        state = self.layer2(state)\n",
    "        state = self.layer3(state)\n",
    "        state = self.layer4(state)\n",
    "        assert a0 == state.shape[0], 'batch size should stay equal'\n",
    "        state = state.reshape(a0, -1)\n",
    "        state = self.layer5(state)\n",
    "        return out5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Screen Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAADECAYAAACGNXroAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAATbUlEQVR4nO3dfbRcVXnH8e8vNwlvwYSYa4xJICABTH1JMAVcWo1ANNoirlWrYgsBUVyrWKCLiqBdCq20YKtIl9XKKiIF5UV5iylqMCa2YAtcQhBIgAQEEkzIJSYGymuSp3+cfZOZYSZ3uHfunNnc32ets+7Z+5w55zkv89w9+5w5o4jAzMzyM6LsAMzMbGCcwM3MMuUEbmaWKSdwM7NMOYGbmWXKCdzMLFNO4NZ2kk6UdGvZcXQS7xMbCCfwVxlJj0p6TtIzFcM3y46rbJLOlXTlEC5/qaRPDdXyzeoZWXYANiSOiYiflx1ETiQJUERsLzuWoSBpZERsLTsOay23wIcRSd+WdF1F+UJJi1XYR9JCSb2SNqXxKRXzLpX0FUm/Sq36H0t6raTvS9oi6U5J0yrmD0mnSXpE0lOS/klS3fNN0iGSbpH0O0kPSvroLrZhrKRLJa2T9ESKqUvSaEnLJf1Vmq9L0m2SviRpHvAF4GMp9nsqtul8SbcBzwIHSDpJ0kpJT6fYP1Oz/mPTerZIeljSPEnnA38EfLPyE8+utivtuwVpOXcAb9zFNu8u6UpJGyVtTvt6Ypo2XtJlkn6bjtuNqX6OpLWSPi9pPXCZpBGSzk5xb5R0raTxFes5Ih3fzZLukTSn5vj/fdqnT0taJGlCo5itTSLCw6toAB4Fjm4wbU/gIeBEioTzFDAlTXst8Kdpnr2BHwI3Vrx2KbCaItGMBVakZR1N8UnuP4DLKuYPYAkwHtg3zfupNO1E4NY0vhewBjgpLWdWimtGg224AfhOet3rgDuAz6RpbwY2AW8Cvgj8L9CVpp0LXFmzrKXA48AfpHWPAv44baOA91Ak9kPT/IcBvwfmUjR+JgOHVCzrUxXL3uV2AVcD16b53gw80bdP6mzzZ4Afp2PTBbwdeE2a9p/ANcA+Kf73pPo5wFbgQmA3YA/g9LRPpqS67wBXpfknAxuBD6Ztm5vK3RXb9zBwUFrWUuCCss/34T6UHoCHFh/QIoE/A2yuGD5dMf1w4HfAY8Bxu1jOTGBTRXkp8MWK8teAn1SUjwGWV5QDmFdR/ktgcRo/kZ0J/GPAf9es+zvAl+vENBF4Adijou44YElF+UzgQYpEPr2i/lzqJ/C/62d/3gicXhHXRQ3mW0p1Am+4XSkJv0RK/mnaP9A4gX8S+BXw1pr6ScB2YJ86r5kDvAjsXlG3Ejiq5vUvUfyD+TxwRc0yfgbMr9i+v605nj8t+3wf7oP7wF+dPhwN+sAj4nZJj1C0Xq/tq5e0J3ARMI+iNQewt6SuiNiWyk9WLOq5OuUxNatbUzH+GPCGOiHtBxwuaXNF3UjgigbzjgLWFV3WQNFarFzP5cD5wHURsarOMmpVvhZJH6BIsgelZe8J3JsmTwVubmKZfbE22q7uNF67fxq5Iq37aknjgCspPmFMBX4XEZsavK43Ip6viekGSZX9/Nso/jHuB/yZpGMqpo2i+BTVZ33F+LO8/HhbmzmBDzOSTqX4+Pxb4CzgH9OkM4GDgcMjYr2kmcDdFF0JAzUVuD+N75vWWWsN8MuImNvE8tZQtMAnROMLct8CFgLvl/SuiOi7Na/RYzd31EvaDbgOOAG4KSJeSn3KfftgDY37qmuX33C7JHVRdG9MBR5I1fs2WC4R8RJwHnBeus5wM8WnjJuB8ZLGRcTmJmP6ZETcViemNRQt8E83isM6jy9iDiOSDgK+AvwFcDxwVkrUUPR7PwdsThe2vtyCVX4uXRydStH/ek2deRYCB0k6XtKoNPyhpDfVzhgR64BFwNckvSZdlHujpPek7Tueon/4ROA04HJJfa3EJ4FpjS6kJqMp/rn1AltTa/x9FdMvBU6SdFRa92RJh1Qs/4Bmtit9orkeOFfSnpJmAPMbBSXpvZLekhL/Fopuj+1pf/wE+Fbaz6MkvXsX2/dvwPmS9kvL7ZZ0bJp2JXCMpPeruAC8e7oQOqXh0qx0TuCvTj9W9X3gN0gaSfEmvTAi7kndC18Arkgtz29QXJx6iuJC109bEMdNwF3AcoqLbZfWzhART1MkyY9TtNDXs/PCWz0nUCTaFRT93D8CJknaN23DCRHxTET8AOih6BaC4qIswEZJy+otOMVyGkXX0ibgE8CCiul3UFyUvIjiYuYvKboeAC4GPpLuBPmXJrbrsxRdEOuB7wGXNdhegNen7dxC0Y/9S3Z2MR1PkdAfADYAZ+xiORen7Vkk6WmK43x42rY1wLEU50QvRWv9czhHdDSlCxJmLSUpKC4iri47FrNXK/93NTPLlBO4mVmm3IViZpapQbXA09eIH5S0WtLZrQrKzMz6N+AWeLql6SGKr9yuBe6k+GbfitaFZ2ZmjQzmizyHAasj4hEASVdT3IbUMIFPmDAhpk2bNohVmpkNP3fddddTEdFdWz+YBD6Z6q8CryXdU9rItGnT6OnpGcQqzcyGH0l1H7Uw5HehSDpFUo+knt7e3qFenZnZsDGYBP4ExbMc+kxJdVUi4pKImB0Rs7u7X/YJwMzMBmgwCfxOYLqk/SWNpvjK8IJ+XmNmZi0y4D7wiNgq6bMUzwzuAr4bEff38zIzM2uRQT1ONiJupvnnI5uZWQv5eeA2fFV8B6K/3zLWiK6hjsbsFfOzUMzMMuUEbmaWKSdwM7NMuQ/chq3Hbv3BjvEtj99bNW2v1x9YVT7gyJOrX6zB/FSoWWu4BW5mlikncDOzTDmBm5llyn3gNmw9v3ndjvFnNz5eNW302Jrn9rjP2zqQW+BmZplyAjczy5QTuJlZptwHbsNW5fNNNKL6rSC5bWOdz2epmVmmnMDNzDLlBG5mlikncDOzTDmBm5llygnczCxTTuBmZplyAjczy5QTuJlZppzAzcwy5QRuZpYpJ3Azs0w5gZuZZcoJ3MwsU07gZmaZ6jeBS/qupA2S7quoGy/pFkmr0t99hjZMMzOr1UwL/HvAvJq6s4HFETEdWJzKZmbWRv3+Ik9E/JekaTXVxwJz0vjlwFLg860MzKzVYvu2qvLW555uOO+oPcYOdThmgzbQPvCJEbEuja8HJrYoHjMza9KgL2JGRADRaLqkUyT1SOrp7e0d7OrMzCwZaAJ/UtIkgPR3Q6MZI+KSiJgdEbO7u7sHuDozM6s10AS+AJifxucDN7UmHLP2ie3bdwy1NKKrajDrRM3cRngV8D/AwZLWSjoZuACYK2kVcHQqm5lZGzVzF8pxDSYd1eJYzMzsFfA3Mc3MMuUEbmaWKSdwM7NMOYGbmWXKCdzMLFNO4GZmmXICNzPLlBO4mVmmnMDNzDLlBG5mlikncDOzTDmBm5llqt+HWZkNTw1/o8SsY7gFbmaWKSdwM7NMOYGbmWXKfeA2bNT+dFps39Zw3hFdo4c6HLNBcwvczCxTTuBmZplyF4oNG9u3vlBV3vbisxUlVU0btedr2hCR2eC4BW5mlikncDOzTDmBm5llygnczCxTTuBmZplyAjczy5QTuJlZppzAzcwy1W8ClzRV0hJJKyTdL+n0VD9e0i2SVqW/+wx9uGZm1qeZFvhW4MyImAEcAZwqaQZwNrA4IqYDi1PZzMzapN8EHhHrImJZGn8aWAlMBo4FLk+zXQ58eIhiNDOzOl5RH7ikacAs4HZgYkSsS5PWAxNbG5qZme1K0wlc0hjgOuCMiNhSOS0iggY/IijpFEk9knp6e3sHFayZme3UVAKXNIoieX8/Iq5P1U9KmpSmTwI21HttRFwSEbMjYnZ3d3crYjYzM5q7C0XApcDKiPh6xaQFwPw0Ph+4qfXhmZlZI808D/ydwPHAvZKWp7ovABcA10o6GXgM+OiQRGhmZnX1m8Aj4lZqn3a/01GtDcfMzJrlb2KamWXKP6lmVlfdm6rMOopb4GZmmXICNzPLlBO4mVmm3Aduw8b2l56vLm99ccd48XWHnUbuPqYtMZkNhlvgZmaZcgI3M8uUu1Bs2Ni+7cWqcmzftrNQ81W1EaN3b0NEZoPjFriZWaacwM3MMuUEbmaWKfeB2zDS6JlsdYS/Sm+dzy1wM7NMOYGbmWXKCdzMLFNO4GZmmXICNzPLlBO4mVmmnMDNzDLlBG5mlikncDOzTDmBm5llygnczCxTTuBmZplyAjczy5QTuJlZpvw4WRtGmn9EbO2v1Jt1IrfAzcwy1W8Cl7S7pDsk3SPpfknnpfr9Jd0uabWkaySNHvpwzcysTzMt8BeAIyPibcBMYJ6kI4ALgYsi4kBgE3DykEVpZmYv028feEQE8EwqjkpDAEcCn0j1lwPnAt9ufYhmrREvPFdd3vbCjvERI6rbMruNGd+WmMwGo6k+cEldkpYDG4BbgIeBzRGxNc2yFpjc4LWnSOqR1NPb29uCkM3MDJpM4BGxLSJmAlOAw4BDml1BRFwSEbMjYnZ3d/fAojQzs5d5RbcRRsRmSUuAdwDjJI1MrfApwBNDEaANb8uWLasqn3XWWQNe1oGv262qfNKc6TsLNbcNnvk3Z1aVVz/5/IDX+9WvfrWqfOihhw54WWaVmrkLpVvSuDS+BzAXWAksAT6SZpsP3DREMZqZWR3NtMAnAZdL6qJI+NdGxEJJK4CrJX0FuBu4dAjjNDOzGs3chfJrYFad+kco+sPNzKwE/iq9dbSNGzdWlRcvXjzgZa3Z74Cq8kFvOWfHeFDdB/6L206oKq967JEBr7d2G8xaxV+lNzPLlBO4mVmmnMDNzDLlPnDraCNHtu4U7Rq9d1V5e9e4xusdtXfDaa9UK7fBrJJb4GZmmXICNzPLlBO4mVmm2to5t3XrVvxEQnslNm3a1LJlPbXhoaryjT88reG8vb2rWrbe2m3we8BaxS1wM7NMOYGbmWWqrV0okhg92j+dac1r5S14vb+v/kWe3p4lLVv2rtRug98D1ipugZuZZcoJ3MwsU07gZmaZamsfeFdXF2PHjm3nKi1zY8aMKTuEQavdBr8HrFXcAjczy5QTuJlZppzAzcwy5edcWkfbtm1b2SEM2qthG6wzuQVuZpYpJ3Azs0w5gZuZZcp94NbRJkyYUFWeO3duSZEMXO02mLWKW+BmZplyAjczy5S7UKyjzZo1q6q8aNGikiIx6zxugZuZZcoJ3MwsU07gZmaZUkS0b2VSL/AYMAF4qm0rbo5jao5jal4nxuWYmtNpMe0XEd21lW1N4DtWKvVExOy2r3gXHFNzHFPzOjEux9ScToypHnehmJllygnczCxTZSXwS0pa7644puY4puZ1YlyOqTmdGNPLlNIHbmZmg+cuFDOzTLU1gUuaJ+lBSaslnd3OddfE8V1JGyTdV1E3XtItklalv/u0OaapkpZIWiHpfkmnlx2XpN0l3SHpnhTTeal+f0m3p+N4jaTR7YqpIrYuSXdLWtgJMUl6VNK9kpZL6kl1ZZ9T4yT9SNIDklZKekcHxHRw2kd9wxZJZ3RAXH+dzvH7JF2Vzv3Sz/P+tC2BS+oC/hX4ADADOE7SjHatv8b3gHk1dWcDiyNiOrA4ldtpK3BmRMwAjgBOTfunzLheAI6MiLcBM4F5ko4ALgQuiogDgU3AyW2Mqc/pwMqKcifE9N6ImFlx+1nZ59TFwE8j4hDgbRT7q9SYIuLBtI9mAm8HngVuKDMuSZOB04DZEfFmoAv4OJ1xTu1aRLRlAN4B/KyifA5wTrvWXyeeacB9FeUHgUlpfBLwYFmxpRhuAuZ2SlzAnsAy4HCKLziMrHdc2xTLFIo3+ZHAQkAdENOjwISautKOHTAW+A3pOlcnxFQnxvcBt5UdFzAZWAOMp3jA30Lg/WWfU80M7exC6dtJfdamuk4xMSLWpfH1wMSyApE0DZgF3F52XKmrYjmwAbgFeBjYHBFb0yxlHMdvAGcB21P5tR0QUwCLJN0l6ZRUV+ax2x/oBS5LXU3/LmmvkmOq9XHgqjReWlwR8QTwz8DjwDrg98BdlH9O9csXMeuI4l9uKbfnSBoDXAecERFbyo4rIrZF8XF3CnAYcEg7119L0p8AGyLirjLjqONdEXEoRRfhqZLeXTmxhGM3EjgU+HZEzAL+j5puiZLP89HAh4Af1k5rd1ypv/1Yin96bwD24uVdrB2pnQn8CWBqRXlKqusUT0qaBJD+bmh3AJJGUSTv70fE9Z0SF0BEbAaWUHyUHCep71ny7T6O7wQ+JOlR4GqKbpSLS46prxVHRGyg6NM9jHKP3VpgbUTcnso/okjoHXE+UfyjWxYRT6ZymXEdDfwmInoj4iXgeorzrNRzqhntTOB3AtPTld3RFB+fFrRx/f1ZAMxP4/Mp+qDbRpKAS4GVEfH1TohLUrekcWl8D4o++ZUUifwjZcQUEedExJSImEZxDv0iIv68zJgk7SVp775xir7d+yjx2EXEemCNpINT1VHAijJjqnEcO7tPoNy4HgeOkLRneh/27avSzqmmtbPDHfgg8BBFP+oXy+r4pzhx1gEvUbRUTqboR10MrAJ+Doxvc0zvovjY+GtgeRo+WGZcwFuBu1NM9wFfSvUHAHcAqyk+Au9W0nGcAywsO6a07nvScH/fud0B59RMoCcdvxuBfcqOKcW1F7ARGFtRV/a+Og94IJ3nVwC7dcp5vqvB38Q0M8uUL2KamWXKCdzMLFNO4GZmmXICNzPLlBO4mVmmnMDNzDLlBG5mlikncDOzTP0/hE6X8+NqXbkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "resize = T.Compose([T.ToPILImage(),\n",
    "                    T.Resize(40, interpolation=Image.CUBIC),\n",
    "                    T.ToTensor()])\n",
    "\n",
    "\n",
    "def get_cart_location(screen_width):\n",
    "    world_width = env.x_threshold * 2\n",
    "    scale = screen_width / world_width\n",
    "    return int(env.state[0] * scale + screen_width / 2.0)  # MIDDLE OF CART\n",
    "\n",
    "def get_screen():\n",
    "    # Returned screen requested by gym is 400x600x3, but is sometimes larger\n",
    "    # such as 800x1200x3. Transpose it into torch order (CHW).\n",
    "    #print('look', env.render(mode='rgb_array'))\n",
    "    screen = env.render(mode='rgb_array').transpose((2, 0, 1))\n",
    "    # Cart is in the lower half, so strip off the top and bottom of the screen\n",
    "    _, screen_height, screen_width = screen.shape\n",
    "    screen = screen[:, int(screen_height*0.4):int(screen_height * 0.8)]\n",
    "    view_width = int(screen_width * 0.6)\n",
    "    cart_location = get_cart_location(screen_width)\n",
    "    if cart_location < view_width // 2:\n",
    "        slice_range = slice(view_width)\n",
    "    elif cart_location > (screen_width - view_width // 2):\n",
    "        slice_range = slice(-view_width, None)\n",
    "    else:\n",
    "        slice_range = slice(cart_location - view_width // 2,\n",
    "                            cart_location + view_width // 2)\n",
    "    # Strip off the edges, so that we have a square image centered on a cart\n",
    "    screen = screen[:, :, slice_range]\n",
    "    # Convert to float, rescale, convert to torch tensor\n",
    "    # (this doesn't require a copy)\n",
    "    screen = np.ascontiguousarray(screen, dtype=np.float32) / 255\n",
    "    screen = torch.from_numpy(screen)\n",
    "    # Resize, and add a batch dimension (BCHW)\n",
    "    return resize(screen).unsqueeze(0).to(device)\n",
    "\n",
    "\n",
    "env.reset()\n",
    "plt.figure()\n",
    "plt.imshow(get_screen().cpu().squeeze(0).permute(1, 2, 0).numpy(),\n",
    "           interpolation='none')\n",
    "plt.title('Example extracted screen')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## More Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get screen size so that we can initialize layers correctly based on shape\n",
    "# returned from AI gym. Typical dimensions at this point are close to 3x40x90\n",
    "# which is the result of a clamped and down-scaled render buffer in get_screen()\n",
    "init_screen = get_screen()\n",
    "_, _, screen_height, screen_width = init_screen.shape\n",
    "\n",
    "# Get number of actions from gym action space\n",
    "n_actions = env.action_space.n\n",
    "\n",
    "\n",
    "def select_action(state, policy_net_parent=None):\n",
    "    global steps_done\n",
    "    sample = random.random()\n",
    "    eps_threshold = EPS_END + (EPS_START - EPS_END) * \\\n",
    "        math.exp(-1. * steps_done / EPS_DECAY)\n",
    "    steps_done += 1\n",
    "    if sample > eps_threshold:\n",
    "        with torch.no_grad():\n",
    "            # t.max(1) will return largest column value of each row.\n",
    "            # second column on max result is index of where max element was\n",
    "            # found, so we pick action with the larger expected reward.\n",
    "            return policy_net(state).max(1)[1].view(1, 1)\n",
    "    else:\n",
    "        if policy_net_parent and random.random() < 0.5:\n",
    "            return policy_net_parent(state).max(1)[1].view(1, 1)\n",
    "        else:\n",
    "            return torch.tensor([[random.randrange(n_actions)]], device=device, dtype=torch.long)\n",
    "\n",
    "def plot_durations():\n",
    "    plt.figure(2)\n",
    "    plt.clf()\n",
    "    durations_t = torch.tensor(episode_durations, dtype=torch.float)\n",
    "    plt.title('Training...')\n",
    "    plt.xlabel('Episode')\n",
    "    plt.ylabel('Duration')\n",
    "    plt.plot(durations_t.numpy())\n",
    "    # Take 100 episode averages and plot them too\n",
    "    if len(durations_t) >= 100:\n",
    "        means = durations_t.unfold(0, 100, 1).mean(1).view(-1)\n",
    "        means = torch.cat((torch.zeros(99), means))\n",
    "        plt.plot(means.numpy())\n",
    "\n",
    "    plt.pause(0.001)  # pause a bit so that plots are updated\n",
    "    if is_ipython:\n",
    "        display.clear_output(wait=True)\n",
    "        display.display(plt.gcf())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimize Model Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dqn_model(non_final_mask, non_final_next_states):\n",
    "    next_state_values = torch.zeros(BATCH_SIZE, device=device)\n",
    "    next_state_values[non_final_mask] = target_net(non_final_next_states).max(1)[0].detach()\n",
    "    return next_state_values\n",
    "\n",
    "def ddqn_model(non_final_mask, non_final_next_states):\n",
    "    next_state_values = torch.zeros(BATCH_SIZE, device=device)\n",
    "    a_max = target_net(non_final_next_states).argmax(1)[0].detach()\n",
    "    predictions = policy_net(non_final_next_states)\n",
    "    next_state_values[non_final_mask] = predictions[:,a_max]\n",
    "    return next_state_values\n",
    "\n",
    "def optimize_model():\n",
    "    if len(memory) < BATCH_SIZE:\n",
    "        return\n",
    "    transitions = memory.sample(BATCH_SIZE)\n",
    "    # Transpose the batch (see https://stackoverflow.com/a/19343/3343043 for\n",
    "    # detailed explanation). This converts batch-array of Transitions\n",
    "    # to Transition of batch-arrays.\n",
    "    batch = Transition(*zip(*transitions))\n",
    "\n",
    "    # Compute a mask of non-final states and concatenate the batch elements\n",
    "    # (a final state would've been the one after which simulation ended)\n",
    "    non_final_mask = torch.tensor(tuple(map(lambda s: s is not None,\n",
    "                                          batch.next_state)), device=device, dtype=torch.bool)\n",
    "    non_final_next_states = torch.cat([s for s in batch.next_state\n",
    "                                                if s is not None])\n",
    "    state_batch = torch.cat(batch.state)\n",
    "    action_batch = torch.cat(batch.action)\n",
    "    reward_batch = torch.cat(batch.reward)\n",
    "\n",
    "    # Compute Q(s_t, a) - the model computes Q(s_t), then we select the\n",
    "    # columns of actions taken. These are the actions which would've been taken\n",
    "    # for each batch state according to policy_net\n",
    "    state_action_values = policy_net(state_batch).gather(1, action_batch)\n",
    "\n",
    "    # Compute V(s_{t+1}) for all next states.\n",
    "    # Expected values of actions for non_final_next_states are computed based\n",
    "    # on the \"older\" target_net; selecting their best reward with max(1)[0].\n",
    "    # This is merged based on the mask, such that we'll have either the expected\n",
    "    # state value or 0 in case the state was final.\n",
    "    \n",
    "    #next_state_values = dqn_model(non_final_mask, non_final_next_states) #dqn\n",
    "    next_state_values = ddqn_model(non_final_mask, non_final_next_states) #ddqn\n",
    "    \n",
    "    # Compute the expected Q values\n",
    "    expected_state_action_values = (next_state_values * GAMMA) + reward_batch\n",
    "\n",
    "    # Compute Huber loss\n",
    "    loss = F.smooth_l1_loss(state_action_values, expected_state_action_values.unsqueeze(1))\n",
    "\n",
    "    # Alternative loss (l2?)\n",
    "#     loss = torch.mean((expected_state_action_values - state_action_values)**2)\n",
    "    \n",
    "    # Optimize the model\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    for param in policy_net.parameters():\n",
    "        param.grad.data.clamp_(-1, 1)\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "GAMMA = 0.999\n",
    "EPS_START = 0.9\n",
    "EPS_END = 0.05\n",
    "EPS_DECAY = 200\n",
    "TARGET_UPDATE = 10\n",
    "POP_SIZE = 3\n",
    "NUM_EPS = 50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updated Reward: tensor([1.])\n",
      "Updated Reward: tensor([1.])\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-ed4f7282fd35>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m             \u001b[0;31m# Perform one step of the optimization (on the target network)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 46\u001b[0;31m             \u001b[0moptimize_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     47\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m \u001b[0;31m#                 episode_durations.append(t + 1)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-04de5eb49b5f>\u001b[0m in \u001b[0;36moptimize_model\u001b[0;34m()\u001b[0m\n\u001b[1;32m     55\u001b[0m     \u001b[0;31m# Optimize the model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m     \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m     \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     58\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mparam\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpolicy_net\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m         \u001b[0mparam\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclamp_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/cs6476/miniconda3/lib/python3.8/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    183\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m         \"\"\"\n\u001b[0;32m--> 185\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    186\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/cs6476/miniconda3/lib/python3.8/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m    123\u001b[0m         \u001b[0mretain_graph\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 125\u001b[0;31m     Variable._execution_engine.run_backward(\n\u001b[0m\u001b[1;32m    126\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m         allow_unreachable=True)  # allow_unreachable flag\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\n",
    "# model_pnets = []\n",
    "best_pnet = None\n",
    "best_dur = 0\n",
    "update_model = False\n",
    "\n",
    "for _ in range(POP_SIZE):\n",
    "\n",
    "    policy_net = DQN(screen_height, screen_width, n_actions).to(device)\n",
    "    target_net = DQN(screen_height, screen_width, n_actions).to(device)\n",
    "    policy_net.load_state_dict(target_net.state_dict())\n",
    "    target_net.eval()\n",
    "        \n",
    "    optimizer = optim.RMSprop(policy_net.parameters())\n",
    "    memory = ReplayMemory(10000)\n",
    "    \n",
    "    steps_done = 0\n",
    "#     episode_durations = []\n",
    "\n",
    "    for i_episode in range(NUM_EPS):\n",
    "        # Initialize the environment and state\n",
    "        env.reset()\n",
    "        last_screen = get_screen()\n",
    "        current_screen = get_screen()\n",
    "        state = current_screen - last_screen\n",
    "        for t in count():\n",
    "            # Select and perform an action\n",
    "            action = select_action(state, best_pnet)\n",
    "            _, reward, done, _ = env.step(action.item())\n",
    "            reward = torch.tensor([reward], device=device)\n",
    "\n",
    "            # Observe new state\n",
    "            last_screen = current_screen\n",
    "            current_screen = get_screen()\n",
    "            if not done:\n",
    "                next_state = current_screen - last_screen\n",
    "            else:\n",
    "                next_state = None\n",
    "\n",
    "            # Store the transition in memory\n",
    "            memory.push(state, action, next_state, reward)\n",
    "\n",
    "            # Move to the next state\n",
    "            state = next_state\n",
    "\n",
    "            # Perform one step of the optimization (on the target network)\n",
    "            optimize_model()\n",
    "            if done:\n",
    "#                 episode_durations.append(t + 1)\n",
    "#                 plot_durations()\n",
    "                \n",
    "                if t + 1 > best_dur:\n",
    "                    print(\"Updated Reward:\", reward)\n",
    "                    best_dur = t + 1\n",
    "                    update_model = True\n",
    "                    \n",
    "                break\n",
    "        # Update the target network, copying all weights and biases in DQN\n",
    "        if i_episode % TARGET_UPDATE == 0:\n",
    "            target_net.load_state_dict(policy_net.state_dict())\n",
    "    \n",
    "#     model_pnets.append(policy_net)\n",
    "    if update_model:\n",
    "        best_pnet = policy_net\n",
    "        update_model = False    \n",
    "    \n",
    "\n",
    "print('Complete')\n",
    "env.render()\n",
    "env.close()\n",
    "plt.ioff()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}