ex2_regularized_logistic_regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Test1</th>\n",
       "      <th>Test2</th>\n",
       "      <th>Result</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.051267</td>\n",
       "      <td>0.69956</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.092742</td>\n",
       "      <td>0.68494</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.213710</td>\n",
       "      <td>0.69225</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.375000</td>\n",
       "      <td>0.50219</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.513250</td>\n",
       "      <td>0.46564</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Test1    Test2  Result\n",
       "0  0.051267  0.69956       1\n",
       "1 -0.092742  0.68494       1\n",
       "2 -0.213710  0.69225       1\n",
       "3 -0.375000  0.50219       1\n",
       "4 -0.513250  0.46564       1"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "data = pd.read_csv('D:/ProgrammingBook/AndrewNg/ex2data2.txt', sep=',', header=None,\n",
    "                   names = [\"Test1\", \"Test2\", \"Result\"])\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(58, 3) (60, 3)\n"
     ]
    }
   ],
   "source": [
    "positive = data[data[\"Result\"]==1]\n",
    "negative = data[data[\"Result\"]==0]\n",
    "print(positive.shape, negative.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtUAAAHVCAYAAAA+Wwf2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X+MJOdd5/HPd7xnRyMEvY6dxMSeGVtnfgTCOTAyiEjkd2JOJ9snQthcR2wg0SpAuNYiTnFuONLyMXeGO2mZEwgYQoiBUezEJy4LJLIcOzl0AgePhRMnjhxvzM5mWRMvcSYILTjY870/nqpMTW/3THdXdf146v2SRjNVXT3zdE319Gee/j7PY+4uAAAAANObq7oBAAAAQNMRqgEAAICcCNUAAABAToRqAAAAICdCNQAAAJAToRoAAADIiVANAAAA5ESoBgAAAHIiVAMAAAA5Haq6AdO44oorfGlpqepmAAAAIGIPP/zw37v7leMc28hQvbS0pM3NzaqbAQAAgIiZ2da4x1L+AQAAAOREqAYAAAByIlQDAAAAORGqAQAAgJwI1QAAAEBOhGoAAAAgJ0I1AAAAkBOhGgAAAMiJUA0AAADkRKgGAAAAciJUAwAAADkRqgEAAICcCNUAAABAToRqAAAAICdCNQAAAJAToRoAAADIiVANIHru+28DAJAXoRpA1Pp96fjx3SDtHrb7/SpbBQCIDaEaQLTcpe1taW1tN1gfPx62t7fpsQYAFOdQ1Q0AgFkxk06cCF+vrYUPSer1wn6z6toGAIiLeQO7apaXl31zc7PqZgBoCHdpLvO+3M4OgRoAcDAze9jdl8c5lvIPAFFLSz6ysjXWaCYGnwKoG0I1gGhla6h7vdBD3evtrbFG8zD4FEAdUVMNIFpmUqezt4Y6rbHudCgBaaLs4FMp/D6z/zi583sFUA1qqgFEbzBoEbyaLfsORIrBpwBmYZKaakI1AKBxGHwKoAwMVAQARIvBpwDqiFANAGgMBp8CqCsGKgIAGoPBpwDqippqAEDjMPgUQBmoqQYARG0wQBOoAVSNUA0AAADkRKgGAAAAciJUAwAAADkRqgEAAICcCNUAAABAToRqAAAAICdCNQAAAJAToRoAAADIiVCN6A0uGtrARURRY1xfAACpoFBtZh8ws6fN7HMjbjcz+19mdsrMPmtm35+57aiZPZF8HC2iPUCq35eOH98NOu5hu9+vslUoWlXBlusLAJAqqqf6g5Ju2uf2H5V0ffJxTNJvSZKZXS7pfZJ+UNKNkt5nZocLahNazl3a3pbW1naDz/HjYXt7mx7FWFQVbLm+AABZh4r4Ju7+52a2tM8ht0j6A3d3SQ+aWcfMrpL0akn3ufszkmRm9ymE8w8V0S60m5l04kT4em0tfEhSrxf2m1XXNhQjG2yl8HtNg22vF26f1e+Z6wsAkGVeUHdKEqr/1N2/d8htfyrpDnf/f8n2/ZLeoxCqX+Duv5Ls/y+S/snd/+eQ73FMoZdbCwsLP7C1tVVIuxE/d2ku857Mzg6BJybZHuJUmcGW6wsA4mVmD7v78jjHljVQcdhLjO+z/+Kd7uvuvuzuy1deeWWhjUO80sCVlS0VQPNle4xTZQZqri8AgFReqD4r6ZrM9tWSzu2zH8gt24PZ64UexF5vbw0smq+qYMv1BQDIKqSmegwnJb3bzO5SGJT4dXd/yszulfTfMoMT3yjpvSW1CZEzkzqdvaUAaY9mp8Nb9DEYDLbZmmpptj3WXF8AgKxCaqrN7EMK9dFXSPqKwowe/0qS3P23zcwk/YbCIMQLkn7K3TeT+/60pP+cfKtVd//9g37e8vKyb25u5m432mFwsNosB6+hfP1+GKyYBts0aHc65Uxtx/UFAPGapKa6sIGKZSJUAwXY2JBWVqQzZ6SFBWl1Vep2q27VVAi2AIBZmCRUl1X+AaBONjakY8ekCxfC9tZW2JYaGawHAzSBGgBQNpYpB9poZWU3UKcuXAj7AQDAxAjVQBudOTPZfgAAsC9CNdBGCwuT7QcAAPsiVANttLoqzc/v3Tc/H/YDBxgc397A8e4AUDhCNdBG3a60vi4tLoZRfYuLYbuBgxRRrn5/7+I26RSGZUxfCAB1RqgG2qrblU6fDksBnj5NoMaB3MOc4NlVI9PFdra36bEG0G5MqQeUgHmUMa46XyvZVSPX1nZXrsyuKgkAbUVPNTBjvF2OcTXhWskG6xSBGgAI1cBM8XY5xtWUayVtV1b2HwEAaCvKPxCFur5lztvlGFcTrpVs0E/blW5L9WknAFTBvIHdC8vLy765uVl1M1AT/X7oyUtf0NMX/k6nPm+bu0tzmfeFdnYIHxiu7tdKE55vAFAUM3vY3ZfHOZbyDzRaE94y5+1yjKsJ10q/v7dHOu1hJ1ADaDtCNRotfUHv9UKQnpvb+9Z01T18g2+X7+zstrVuYQnVatK1Mvi8qvp5hmZh8SDEippqNF4arNO6TqkegVoKbeh09ob8tG6206lHG1EPXCtoA8qHEDNCNRpv1FvmdQnW/f7egZNpWKpD21AvXCuIWbZcT9o70LXXq88Ac2BalH+g0Zryljlvl2NcXCuIVd3L9YC8CNVotFFvmfd6vGUOAHXD4kGIGeUfaDzeMgeAZqh7uR6QBz3ViAJvmQNAvTWlXA+YFj3VAABg5pjhBrFjRUUAAFCawVk+mPUDdcaKigAAoJYo10OsCNUAAABAToRqAAAAICdCNQAAAJAToRoAgBwGx/s3cPw/gAIQqgEAmFK/v3eO5XQu5n6/ylYBqAKhGgCAKbhL29t7Fy9JFzfZ3qbHGmgbQjUAZG1sSEtL0txc+LyxUXWLUFPp4iXpqoBzc7urBbLsNtA+LP4CAKmNDenYMenChd198/PS+rrU7VbXLtSaewjUqZ0dAjUQCxZ/AYBprKzsDdRS2F5ZqaY9qL205CMrW2MNoD0I1QCqU7dSizNnJtuPVsvWUPd6oYc6LQUhWAPtc6jqBgBoqcFSi62tsC1VV2qxsBDaMWw/MMBM6nT21lCfOBFu63QoAQHahppqANVYWhoeYBcXpdOny25NQE01puC+N0APbgNoLmqqAdRfHUstut0QoBcXQypaXCRQ40CDAZpADbQT5R8AqlHXUotut7Ehus49pnVuGwAUgZ5qIEZ1GwA4zOpqKK3Imp8P+zGxOq/sV+e2AUBRCNVAbNK64K2tkF7SAYB1C9aUWhSmziv71bltAFAkBioCsanjAEDMXDaspuqysl+d2wYA+5lkoCKhGojN3Nzw7j+zMJEuolXnlf3q3LZYULcOFI/ZP4A2GzXQr+oBgJipOq/sV+e2xYK6daB6hGogNgwAbJ06r+xX57bFgrp1oB4KmVLPzG6StCbpEknvd/c7Bm4/Iek1yea8pBe5eye57XlJjya3nXH3m4toE9Ba6UC/lZUw5/PCQgjUDACMVp1X9qtz22KRPadra7u169StA+XKXVNtZpdI+qKkN0g6K+khSW9198dGHP/zkl7h7j+dbP+ju3/LJD+TmmoAuFida2rr3LZYULcOFK/smuobJZ1y9yfd/RuS7pJ0yz7Hv1XShwr4uQCAjDqv7FfntsWAunWgekWE6pdK+nJm+2yy7yJmtijpWkkPZHa/wMw2zexBM7t11A8xs2PJcZvnz58voNkAADRfHevWB38m4R5tUERN9bD+hlFPnyOS7nH35zP7Ftz9nJldJ+kBM3vU3b900Td0X5e0LoXyj7yNBgAgBnWrW+/3wwDJtC1p6O90mI0EcSsiVJ+VdE1m+2pJ50Yce0TSz2V3uPu55POTZvYpSa+QdFGoBgAAw/X7e+vU02BddqDOzkQihTZke9GppUfMigjVD0m63syulfS3CsH5PwweZGbfKemwpL/M7Dss6YK7P2tmV0h6paRfK6BNAAC0Sh3q1pmJBG2Wu6ba3Z+T9G5J90r6gqQPu/vnzex2M8tOj/dWSXf53ulGvlvSppl9RtInJd0xatYQAABQf9lgnSJQow0Kmafa3T8m6WMD+355YLs/5H5/IenlRbQB8Wjq1FtNbTcAFGnUTCQEa8SOFRVRK01darep7QaAItVxJhKgLIRq1EZTl9ptarsrsbEhLS2FFSqWlsI2dnF+0HCjZiLp9VhBE/HLvaJiFVhRMV7ZQJpqwgCXpra7VBsb0rFj0oULu/vm56X1dZZQlzg/iArlcIjFJCsqEqoj1eQ/aE1darep7Z7axoa0siKdOSMtLEirq/uHv6UlaWvr4v2Li9Lp07NqZXNwfmZn0msVABJlL1OOmmlyfW9Tl9ptarunlvaqbm2FB7m1Fbb3K1c4c2ay/Xnb17QyijLPT5tMc60CwBQI1ZFpcn1vUwe4NLXduays7C1TkML2ysro+ywsTLZ/Wk0NUWWdn7aZ5loFgCkUMqUe6qPJE+/XbandcTW13blM06u6ujq8Znh1tdi27Rei6vyWf1nnp214BwBASaipjlST63ubWg/e1HZPZdr63zJqW+fmhr89YBaeCHVG7W/x6lyrzu8bqD1qqluu6fW9dVhqdxpNbfdUVldDL2rWOL2q3W4IMjs74fMsAkSTyyjKOD9tM+21OmtNLVPCRAZfd5vyOozpEKoj08r6XpSv2w1TvS0uhv8eFhfrM/VbXUMUqlHXa5Va7+g1edIATIea6si0sr4X1eh2qw8mw6Rt4m11pOp4rVLrHbXspAFSeB3OdnhFXR7YYtRUR6pV9b0A0DR1rvVGIVgULA7UVKNd9b1ojibOHw3MAmVKY2lyTXL2neIUgTpuhGoA5WBgFrCrrrXeNdL0muSmTxqAyRGqAZSDgVnAXsz2MlKTFzKTmDSgrQjVaA9KD6rFwCwAY0pLJ9IgOje3G1CbUEIxatKAXo9JA2LGQEW0Q1p6MLhaHW+3loeBWQAm1OSFzCQmDYgBAxWBQZQeVI+BWZiRJg9mw2gx1CQzaUC7EKrRDpQeVI+BWZiBpg9mw3DUJKOJWPwF7bCwMLz0oAlLV8ekjotwoLFYYCNeLGSGJqKmGu1ATTUQJRbYiBs1yagaNdXAIEoPgCixwEY9zKqunZpkNAmhGsWp+5R1zAkLRCeGwWxNR107EBCqUQxWywNQMgazVa/pi7QARWKgIoqx35R19AgDmAEGs1Uve87X1nZr26lrRxsxUBHFmJsb3iVhFrqPAGBGGMxWvaYv0gKMwkBFlG/U1HRMWQdgxg4czFb38R4NR107EBCqUQxWywOapS1Bk/EeM0VdO7CLmmoUI62bXlkJqxQuLIRATT01UD+D87anQVOK7znLeI+Zoq4d2EVNNQC0zdLS8BVGFxfDdJMxYbxHKahrR6yoqQYAjHbmzGT7m4zxHqVgkRaAUA0A7dOmoMl4DwAlIVQDiE9bBuFNq01Bs9uV1tdDaYtZ+Ly+Tj01gMIRqgHU07TBmNkeDta2oNnthlrxnZ3wOdbHCaBSDFRsKAaFIGqDs1NIoSd1nODXpkF4AICZYqBi5Pr9vfN/pvOE9vtVtgoo0H7ToB2kTYPwZo0yGgAYG6G6Ydyl7e29E+unE+9vbzPRPiKRJxi3aRDeLFFGAwATIVQ3TDqxfrpi1dzc7kpW6cT7QOPlCcZtGoQ3S3neLQCAFiJUN1B2xaoUgRpRyROM2zYIb1aKLqOhlARA5AjVDZSWfGRla6yBxssbjJntIb8iy2goJQHQAoTqhsnWUPd6ITOkpSAEa0SFYFytIstoKCUB0AKHqm4AJmMmdTp7a6jTUpBOhxIQAAVJ/4lZWQklHwsLIVBP888NM7KgYZi2FtMopKfazG4ys8fN7JSZ3Tbk9reb2XkzeyT5eGfmtqNm9kTycbSI9sSu399bQ50Ga6bUawDqStEkRb1bwIwsaBCmrcW0codqM7tE0m9K+lFJL5P0VjN72ZBD73b3G5KP9yf3vVzS+yT9oKQbJb3PzA7nbVMbDP7HXMf/oAdLUVpfmkJdKdqKGVnQEExbizyK6Km+UdIpd3/S3b8h6S5Jt4x53zdJus/dn3H3r0m6T9JNBbQJFeM//SGoK0VbMSMLGoJpa5FHEaH6pZK+nNk+m+wb9GNm9lkzu8fMrpnwvmgQ/tMfgbpStBkDT+uJkrSLMG0tplVEqB52mQ3Gpj+RtOTu3yfpE5LunOC+4UCzY2a2aWab58+fn7qxmD3+0x+BulIAdUJJ2lBMW4tpFRGqz0q6JrN9taRz2QPc/avu/myy+buSfmDc+2a+x7q7L7v78pVXXllAszFL/Kc/BHWlAOqEkrSLMG0t8igiVD8k6Xozu9bMLpV0RNLJ7AFmdlVm82ZJX0i+vlfSG83scDJA8Y3JPjQc/+kPUVVdKW/vAhiGkrSLjJq2ttdj2locLPc81e7+nJm9WyEMXyLpA+7+eTO7XdKmu5+U9B/N7GZJz0l6RtLbk/s+Y2b/VSGYS9Lt7v5M3jahWoP/6Z84sbsttbzHutstt5Y0fXs37Y1K395N2wKgvRYWwt+EYftbrN/fOy91Gqxb+7qFsZk3sOtweXnZNzc3q24G9tHvh0GJ6R+iNGh3OgXPALKxUcziFLFaWhr+orm4GAaLAWivwX+6pVCSxswswDeZ2cPuvjzWsYRqzMrMV6TiBeFgc3PDa27MQrEggHajYwLYF6Ea7UAv7ME4RwAATG2SUF3IMuVAJRhkczBmHAEAoBSEajQX8z4fjJXsALQdMyChJIRqNBe9sONhJTsAbcUCNygRoRrNRS8sAGA/LHCDEjFQEQAAxIkZkJATAxUBAAAYe4MSEaoBAECcGHuDEhGqAQBAnBh7gxIdqroBAAAAM9PtEqJRCnqqAQAAgJwI1QAAAEBOhGoAyIPV2gAAoqYaAKaXrtaWLi6RrtYmUcMJAC1DTzUATIvV2gAACUI1AEzrzJnJ9gMAokWoBoBpsVobACBBqAaAabFaGwAgQagGgGmxWhuAmnPffxvFYfYPAMiD1doA1FS/L21vSydOhP/73aXjx6VOJ9yGYtFTDQAAEBn3EKjX1kKQTgP12lrYT4918QjVAPJh8RMAqB2z0EPd64UgPTcXPvd6uz3XKJZ5A/9VWV5e9s3NzaqbAWBw8RMpDNSjrhgAasE9BOrUzg6BehJm9rC7L49zLD3VAKbH4icAUFtpyUdWWgqC4hGqAUyPxU8AoJayNdS9XuihTktBCNazQajGZKifRRaLnwBALZmFWT6yNdRpjXWnQwnILFBTjfFRP4tBXBMAUGvuewP04Db2R001ZoP6WQxi8RMAqLXBAE2gnh16qjG+ubnhRVhmoVgLAAAgIvRUz0jrl/qkfhYAAGAoQvWY+v29o2XTUbWtWuZzdTXUy2bNz4f9mA4DPwEAiAKhegws9ZmgfrZY6SC/ra1wEW1thW2CNQAAjUNN9ZiyQTrFUp/IZWkpBOlBi4vS6dNltwYAAAyYpKaaUD2BWS/1ybQ3LcPATwAAao2BijMw66U+qdluIQZ+AgAQDUL1GGa91Cc12y3FwE8AAKJxqOoGNMGopT6lYpb6zH6/tbXdum1qtiOXDvBcWZHOnAk91KurDPwEgEhQ1tku1FRPYNZPjlnXbAMAgHL0++Hd5rRzLH0XutOhtLNJqKmekVku9Tnrmm0AAFAOyjrbiVBdA7Ou2QYAYGosUjWxtKwzfS2fm9t9jaesM17UVNfArGu2AQCYSrpI1YULYTtdpEpi/McB0tfy7PoWBOq4UVNdIwxoAADUCotUTY1F4+JQek21md1kZo+b2Skzu23I7b9gZo+Z2WfN7H4zW8zc9ryZPZJ8nCyiPU01y5ptAAAmdubMZPshibLOtspd/mFml0j6TUlvkHRW0kNmdtLdH8sc9teSlt39gpn9jKRfk/QTyW3/5O435G0HAAAo2MLC8J5qFqnaF2Wd7VRETfWNkk65+5OSZGZ3SbpF0jdDtbt/MnP8g5LeVsDPBQAAs7S6uremWmKRqjH1+3vLONNgTaCOVxHlHy+V9OXM9tlk3yjvkPTxzPYLzGzTzB40s1sLaA8AAChCtyutr4caarPweX2dQYpjoqyzXYroqR52iQytFjKzt0lalvSqzO4Fdz9nZtdJesDMHnX3Lw257zFJxyRpgbedAAAoR7dLiAbGUERP9VlJ12S2r5Z0bvAgM3u9pBVJN7v7s+l+dz+XfH5S0qckvWLYD3H3dXdfdvflK6+8soBmAwAAAMUoIlQ/JOl6M7vWzC6VdETSnlk8zOwVkn5HIVA/ndl/2MwuS76+QtIrlanFBgAAAJogd6h29+ckvVvSvZK+IOnD7v55M7vdzG5ODvsfkr5F0kcGps77bkmbZvYZSZ+UdMfArCFAtVhJDAAAjIHFX4BRBlcSk8KodwbpAADQCqUv/gKUpsye45WVvYFaCtsrK7P7mQAAoJGKmP0DKMdgz/HWVtiWZtNzzEpiAABgTPRUoznK7jkeNXUjUzoCAIABhGo0R9k9x6uroYY6i5XEAADAEIRqNEfZPcesJAYAAMZEqEZzVNFz3O1Kp09LOzvhM4EaAAAMQahGc9BzDAAAaorZP9As3S4hGgAA1A491QDyYdVJAADoqQaQQ9lzhwMAUFP0VAOYHqtOAgAgiVANIA9WnQQwDsrE0AKEagDTY9VJAAdJy8S2tiT33TIxgjUiQ6gGMD1WnQRwEMrE0BKEagDTY+5wAAehTAwtwewfAPJh7nAA+1lYCCUfw/YDEaGnGgAAzA5lYmgJQjUAAJgdysTQEpR/AACA2aJMDC1ATzUAAACQE6EaAAAAyIlQDQAAAOREqAYAAECtuO+/XUeEagAAANRGvy8dP74bpN3Ddr9fZasORqgGAABALbhL29vS2tpusD5+PGxvb9e7x5op9QAAACD3MJX4qO0ymEknToSv19bChyT1emF/2e2ZBD3VAAAALVenkotssE7VPVBLhGoAAIBWq1vJRfrzs7KBv64I1QAAAC2W9gz3eiFIz82Fz1WUXGQDfa8n7ezstqvuwZpQDQAA0HJ1KbkwkzqdvYE+DfydTr1LQBioCAAA0HKjSi6qCNb9/t5BkmmwrnOgluipBgAAaLU6llwMBui6B2qJnmoAAIBWG1VyIdW/5KJOzOtc8T3C8vKyb25uVt0MAACAaNRhnuq6MbOH3X15nGMp/4jU4P9KDfzfCQAAlKiJJRd1QqiOUJ0mcAcAAGgDQnVk6jaBeyttbEhLS2Giz6WlsA0AAKLGQMXIZAcXrK2FD6maCdxbaWNDOnZMunAhbG9thW1J6naraxcAAJgpeqojVJcJ3FtpZWU3UKcuXAj7y0AvOQAAlSBUR2jUBO6UfpTgzJnJ9hcp7SXf2gq/7LSXnGANAMDMEaojU8cJ3FtlYWGy/UWqupccAFB7zA42O4TqyIyawL3XYwL3UqyuSvPze/fNz4f9s1ZlLzmA4lDGhRlhdrDZYqBihPr9vRO2p8GaQF2CdDDiykoIswsLIVCXMUhxYSGUfAzbD6AZGOyMGcnODiaFXJB9Z5uFXvIrpKfazG4ys8fN7JSZ3Tbk9svM7O7k9k+b2VLmtvcm+x83szcV0R7UZAL3tva2dLvS6dOh9ub06fJeCKvsJQdQDMq4MCPZd67X1sJLcxqo6XgrRu5lys3sEklflPQGSWclPSTpre7+WOaYn5X0fe7+LjM7Iunfu/tPmNnLJH1I0o2Svl3SJyR9h7s/v9/PZJnyBhjsbZFCwFtfp7dlljY2quklB1CMubnhRa5m4R91ICf3cJmldnYI1Pspe5nyGyWdcvcn3f0bku6SdMvAMbdIujP5+h5JrzMzS/bf5e7PuvvfSDqVfD80Hb0t1aiqlxxAMaoc7IzoMTvYbBURql8q6cuZ7bPJvqHHuPtzkr4u6YVj3hdNxKA5AJgcZVyYEWYHm70iQvWwNw0GfzWjjhnnvuEbmB0zs00z2zx//vyETUTp6G2JW1vr5YFZ63ZDmdziYnhPfnGRsjkUgtnBZq+I2T/OSroms321pHMjjjlrZockfZukZ8a8ryTJ3dclrUuhprqAdmOWVleH11TT29J8zE4AzFa3y3MJM8HsYLNVRE/1Q5KuN7NrzexSSUcknRw45qSko8nXb5b0gIcRkiclHUlmB7lW0vWS/qqANqFq9LbEi3p5AFNg0ZF6qMXsYJHK3VPt7s+Z2bsl3SvpEkkfcPfPm9ntkjbd/aSk35P0h2Z2SqGH+khy38+b2YclPSbpOUk/d9DMH2gQelviRL08gAn1+2GO5LRXNK3v7XRYeATxKGTxF3f/mKSPDez75czX/yzpx0fcd1USNQFAU7DIDIAJsOgI2oIVFQFMhnp5ABNI63alEKTTcM2iI4hNISsqAmgR6uUBTCgbrFMEasSGUA1gciwyA2ACLDqCNiBUAxFjtD2AqrHoCNqCmmogUoy2B1AHoxYdkVh0BHEhVCO3wZHbjOSuHqPtAdQJi46gDQjVyIXe0HpitD2AumHREcSOmmpMLdsbmtbFpb2h29vUyVWN0fYAAJSHUN0wdRp4loa2dMDJ3NxueQHhrXqMtgeA5qnT6zwmQ6hukH5/byhKQ1OVZRb0htYTo+0BoHnq+DqP8RGqG6KupRb0htbTqNH2vR6j7QGgjur6Oo/xmTfwt7S8vOybm5tVN6N02SdYqspSi8He0MEZJuixrh4zswBAc9TtdR6SmT3s7stjHUuobhb3ULuc2tmp9onG7B8AABSnbq/zbTdJqGZKvQYZVWpR5X+wzD0KAEAx6vg6j/FRU90QdR54xtyjAADkU+fXeYyHnuqGYJlXAADixet881FT3TAMPAMAIF68ztfLJDXVlH80DKUWAADEi9f55iJUo7VYtQoAABSFUI1WYtUqAABQJEI1WodVqwAAQNEI1Wid7JLda2thkn1WgQSAkm1sSEtL4Y/w0lLYBhqM2T/QWqxaBQAV2diQjh2TLlzY3Tc/L62vS91ude0CBjD7B3CAUatWNfB/TABonpWVvYFaCtsrK9W0BygAoRqtw6pVAFCxM2cm2w80AKEarTNq1apej1WrUCPUmyJmCwuT7QcagJpqtBarVqG2qDdF7LjG0RDUVANjYNUq1Bb1pohdtxsC9OJi+OO7uEigRuPRUw0AdTM3N7y43ywMAgAAlIKeagBoMupNAaBxCNUA2qkSZCpBAAAWsklEQVTOAwFXV0N9adb8fNiPdqvzddsUnEPMyKGqGwAApRscJLW1FbaletR0pm1YWQlTjC0shEBdh7ahOnW/bpuAc4gZoqYaiMHGBgFsEktL4cV00OKidPp02a0BxsN1mx/nEBOapKaanmqg6eh5mRwLT6CJuG7z4xxihqipBpqO6dcmx0BANBHXbX6RncPBYoMGFh9EhVANnpRNR8/L5BgIiCbius0vonPY70vHj+++ZruH7X6/yla1G6G65XhSRmCWPS+xjpJn4Qk0EddtfpGcQ3dpe1taW9t9DT9+PGxvb9M5VhUGKrZY9knY60knTly8zSqDDTCr5X5ZRhgAaiv7Gp7itbt4kwxUJFS3HE/KSMxi9g9GyQNArbmHNxJTOzu8dheNFRUxNrMQoLMI1A3U7Yagu7MTPhfRk0ytNpog1hIl4ABpp1hWtpwT5SNUtxxPSowU2Sh5RCgtUdraCn+00ukkCdaI3GD55s5O+JytsUb5CNUtxpMS+4polDwixXSSaCkzqdPZW6554kTY7nR4t7kqLP7SYqOelBJPSoilslF/lCg1EyvAFqLfD51f6Wt1+hrOa3d1cg1UNLPLJd0taUnSaUlvcfevDRxzg6TfkvStkp6XtOrudye3fVDSqyR9PTn87e7+yEE/l4GKxco+KYdtA0AtMZi2eZhVCA1T5kDF2yTd7+7XS7o/2R50QdJPuvv3SLpJ0q+bWSdz+39y9xuSjwMDdZuUtSjLYIAmUBeDRXWAGaNEqXko2UHE8obqWyTdmXx9p6RbBw9w9y+6+xPJ1+ckPS3pypw/N3osytJs/P6AEkSykEerULKDiOUN1S9296ckKfn8ov0ONrMbJV0q6UuZ3atm9lkzO2Fml+VsTxRYKanZ+P0BJZrFdJKYHWYVQsQOrKk2s09IesmQm1Yk3enuncyxX3P3wyO+z1WSPiXpqLs/mNn3dwpBe13Sl9z99hH3PybpmCQtLCz8wNawOrqIsChLs/H7A4AhqKlGwxRaU+3ur3f37x3y8VFJX0mCcRqQnx7RoG+V9GeSfikN1Mn3fsqDZyX9vqQb92nHursvu/vylVfGXz3Coiw1NsZiE/z+CsCiHkB8KNlBxPKWf5yUdDT5+qikjw4eYGaXSvpjSX/g7h8ZuC0N5KZQj/25nO2JBouy1NSYi03w+8uJRT2AeFGyg0jlDdV3SHqDmT0h6Q3Jtsxs2czenxzzFkk/IuntZvZI8nFDctuGmT0q6VFJV0j6lZztiQKLstTYGCPX+f0VgBkCAAANk2vxF3f/qqTXDdm/Kemdydd/JOmPRtz/tXl+fqxYlKXGxhi5zu+vAMwQAABomFyLv1SlLYu/sChLDU2w2AS/vxxY1AMAUANlLv6CGWJRlnxmsvjKBItN8PvLgUU9AAANQ6hGlGa2+Aoj18vBeQYANEyummqgjrKLr0ihnjk7cDB3GUa3S7grA+cZANAghGpEJzswcG1tN1yz+AoAAJgVBioiWu5h3ZDUzg6BGgAAjI+Bimg9Fl8BAABlIlQjOiy+AgAAykZNNaLD4isAAKBs1FQjWiy+AgAA8qCmGhCLrwAAgPIQqgEAAICcCNUAAABAToRqAAAAICdCNQAAAJAToRpAbQxORtTAyYkAYCL83YsHoRpALfT7exfnSRfx6ferbBUAzA5/9+JCqAZQOXdpe3vvqpfpqpjb2/TcAIgPf/fiw+IvAGoh+4KSyq6KCQCx4e9e/U2y+AuhGkBtuEtzmffPdnZ4YQEQN/7u1RsrKgJonLTHJitbawgAseHvXlwI1QAql30LtNcLPTW93t5aQwCICX/34kOoBlA5M6nT2VtLeOJE2O50RrwVurEhLS2F902XlsI2ADTEVH/3UGvUVAOoDfe9LySD29+0sSEdOyZduLC7b35eWl+Xut2ZtxMAijL23z1UgppqoERM3F+cwReSkS8sKyt7A7UUtldWZtKuXOhRB7CPsf/uofYI1UAOTNxfkTNnJttflbRHfWsrXBxbW2G7rcGafzAARIxQjcK1peeWifsrtLAw2f6qNKlHfdb4B2NibflbCsSCmmoUqt8PgTIddJEGzU4nzt5bJu6vSFNqqufmhichszDUv02WlkKQHrS4KJ0+XXZraq9tf0uBuqKmGpVoY89tOlo7i0Bdgm43BOjFxXCyFxfrF6il5vSol6EpJTs10Ma/pUAM6KlGodrWc9u2x4sJNaVHvQz0VE+Evy1APdBTjcq0qeeWiftxoKb0qJdhdTX8Q5E1Px/24yJt+lsKxIJQjUK1aclVJu7HWLrd0BO7sxM+tzFQS/yDMaE2/S0FYnGo6gYgHoM9tydO7H37MsZeln5/70T9abCO7XFWiYURItLtEqLH0Ma/pUAMCNUozKieWynunlsm7p8dZkBAG7X1bynQdIRqFIqeWxQlOwOCtLe3rtejxxpx428p0DyEahSOnlsUIds7t7a2G66ZAQFtwd9SoFmYUg9ArbmHNVRSOzuECwBAOZhSD0AUmAEBANAUhGoAtcQ84ACAJqGmGkAtMQMCAKBJqKkGUGvMUw0AqAo11QCiwQwIAIAmIFQDAAAAOeUK1WZ2uZndZ2ZPJJ8PjzjueTN7JPk4mdl/rZl9Orn/3WZ2aZ72AAAAAFXI21N9m6T73f16Sfcn28P8k7vfkHzcnNn/q5JOJPf/mqR35GwPAAAAULq8ofoWSXcmX98p6dZx72hmJum1ku6Z5v4AAABAXeQN1S9296ckKfn8ohHHvcDMNs3sQTNLg/MLJW27+3PJ9llJLx31g8zsWPI9Ns+fP5+z2QDQUhsb0tJSWKZyaSlsAwByO3CeajP7hKSXDLlpZYKfs+Du58zsOkkPmNmjkv5hyHEj5/dz93VJ61KYUm+Cnw0AkEKAPnZMunAhbG9thW1J6naraxcQOaYGbYcDe6rd/fXu/r1DPj4q6StmdpUkJZ+fHvE9ziWfn5T0KUmvkPT3kjpmlgb7qyWdy/2IAADDrazsBurUhQthP+LHuxSV6Pf3rgKbrhbb71fZKsxC3vKPk5KOJl8flfTRwQPM7LCZXZZ8fYWkV0p6zMOqM5+U9Ob97g8AKMiZM5PtRzzSdym2tkKqS9+lIFjPlLu0vS2tre0G6+PHw/b29m7QRhxyrahoZi+U9GFJC5LOSPpxd3/GzJYlvcvd32lmPyzpdyTtKIT4X3f330vuf52kuyRdLumvJb3N3Z896OeyoiIATGFpKYSpQYuL0unTZbcGZeJ3X5lskE71etKJE5SANEFpKyq6+1fd/XXufn3y+Zlk/6a7vzP5+i/c/eXu/m+Sz7+Xuf+T7n6ju/9rd//xcQI1AGBKq6vS/PzeffPzYT/qp8hyjaa+SxFByYpZCNBZBOo4saIiALRFtyutr4feSbPweX2dQYp1VHS5xsLCZPvrIJKSlbSnOitbY414EKoBoE263fB2/85O+EygrqeiB5U28V2KCAbWZks/er3wtOv19tZYIx4HTqkHAABKVnS5RvrP08pK+B4LCyFQ1/mfqpqXrIwzTZ6Z1OnsraFOS0E6HUpAYpNroGJVGKgIAIgaAwtrfQ76/TB7RxqU0x7pTmf4VHnMU91cpQ1UBAAAM9DEco2i1fQcTDNN3rAebMSHUA0AwCTKmJGCQaW1PQdpCUdaGz03t1szzawe7Ub5BwAA4xpc6l0Kvac1CHsol3sI1KmdHQJ1jCj/AIApDfYzNLDfYSptfdypsR9/BDNSID+mycMwhGoASPT7e18Y0xfOYQOPYtLWx52a6PHXfEYKzB7T5GEUQjUAaLrBRzFo6+NOTfz4m7iICgo1apq8Xo9p8tqOmmoASGQDVaoNg4/a+rhTEz3+BtVUM43bbHF+22GSmmpCNQBktHXwUVsfd2qix7+xUftFVCadRxnAcAxUBIAptHXwUVsfd2rix1/zpd7bXtIDVIVQDQBq7+Cjwh53GXM3z0CMv3fmUQaqcajqBgBAHYwafCTFPfiokMc9WGe8tRW2pdr14g6K9feePo5snTiBGpgtaqoBIKOtg49yPe6lpRCkBy0uhvKIBojt9972wadAUaipBoApDQaOtgSQXI87grmbY/q9x1jSAjQB5R8AgHwWFob3VDN3cyViLWkB6o5QDQDIZ3V1+NzNq6vVtanl+v29JSxpsCZQA7ND+QeAkQbfJuZtYwzV7YbFTxYXQ2pbXKzlYihtE1NJC9AE9FQDGIrFIzCRbpcQDaDV6KkGcBEWjwAAYDL0VAO4SHZg09ra7rRcTMkFAMBwzFMNYCT3sBpbameHQA0AaA/mqQaQW1rykcUctwAADEeoBnARFo8AAGAy1FQDuAiLRwAAMBlqqgGMlF08Ytg2AAAxo6YaQCFYPAIAgPEQqgEAAICcCNUAAABAToRqAAAAICdCNQAAAJAToRoAAADIiVANAAAA5ESoBgAAAHIiVAMAAAA5EaoBAACAnAjVAAAAQE6EagAAACAnQjUAAJiY+/7bQNsQqgEAwET6fen48d0g7R62+/0qWwVUi1ANAADG5i5tb0tra7vB+vjxsL29TY812utQnjub2eWS7pa0JOm0pLe4+9cGjnmNpBOZXd8l6Yi7/x8z+6CkV0n6enLb2939kTxtAgAAs2MmnUhe1dfWwock9Xphv1l1bQOqZJ7jX0oz+zVJz7j7HWZ2m6TD7v6efY6/XNIpSVe7+4UkVP+pu98zyc9dXl72zc3NqdsNAE3nvje8DG4Ds+YuzWXe797Z4RpEfMzsYXdfHufYvOUft0i6M/n6Tkm3HnD8myV93N0v5Py5ANBa1LOiauk1l5W9JoE2yhuqX+zuT0lS8vlFBxx/RNKHBvatmtlnzeyEmV026o5mdszMNs1s8/z58/laDQANRT0rqpa95nq90EPd6+29JoE2OrCm2sw+IeklQ25ameQHmdlVkl4u6d7M7vdK+jtJl0pal/QeSbcPu7+7ryfHaHl5macsgFainjWg/KU6ZlKns/eaS6/JToffA9orb03145Je7e5PJaH5U+7+nSOO7Un6Hnc/NuL2V0v6RXf/dwf9XGqqAbRdm+tZ+/3QK58GurTntNOhBKZM/GODNiizpvqkpKPJ10clfXSfY9+qgdKPJIjLzEyhHvtzOdsDANFrcz0r5S/1MRigCdRou7yh+g5JbzCzJyS9IdmWmS2b2fvTg8xsSdI1kv7vwP03zOxRSY9KukLSr+RsDwBEre31rGmpQfqY5+Z2z0Wbyl8A1E+u8o+qUP4BoM0of2h3+QuA8kxS/pFr8RcAQPn6/b31q2nvbVtC5ajylzadAwD1wzLlANBAba1nbXv5C4D6oqcaANAYTOcGoK6oqQYANA7TuQEoQ5lT6gEAULq2lr8AqC9CNQAAAJAToRoAAADIiVANAAAA5ESoBgAAAHIiVAMAAAA5EaoBAACAnAjVAAAAQE6EagAAACAnQjUAAACQE6EaAAAAyIlQDQAAAOREqAYAAAByIlQDAAAAORGqAQAAgJwI1QAAAEBOhGoAAAAgJ3P3qtswMTM7L2mr6naU4ApJf191IxqKc5cP5296nLt8OH/T49zlw/mbXsznbtHdrxznwEaG6rYws013X666HU3EucuH8zc9zl0+nL/pce7y4fxNj3MXUP4BAAAA5ESoBgAAAHIiVNfbetUNaDDOXT6cv+lx7vLh/E2Pc5cP5296nDtRUw0AAADkRk81AAAAkBOhGgAAAMiJUF0xM7vczO4zsyeSz4eHHPMaM3sk8/HPZnZrctsHzexvMrfdUP6jqMY45y457vnM+TmZ2X+tmX06uf/dZnZpea2v3pjX3g1m9pdm9nkz+6yZ/UTmttZde2Z2k5k9bmanzOy2IbdfllxLp5Jraylz23uT/Y+b2ZvKbHcdjHHufsHMHkuus/vNbDFz29DncJuMcf7ebmbnM+fpnZnbjibP8yfM7Gi5La/eGOfuROa8fdHMtjO3tfraM7MPmNnTZva5Ebebmf2v5Nx+1sy+P3Nb+647d+ejwg9JvybptuTr2yT96gHHXy7pGUnzyfYHJb256sdR53Mn6R9H7P+wpCPJ178t6Weqfkx1O3+SvkPS9cnX3y7pKUmdZLtV156kSyR9SdJ1ki6V9BlJLxs45mcl/Xby9RFJdydfvyw5/jJJ1ybf55KqH1PNzt1rMn/XfiY9d8n20OdwWz7GPH9vl/QbQ+57uaQnk8+Hk68PV/2Y6nTuBo7/eUkfyGy3/dr7EUnfL+lzI27/t5I+Lskk/ZCkTyf7W3nd0VNdvVsk3Zl8faekWw84/s2SPu7uF2baqmaY9Nx9k5mZpNdKumea+0fiwPPn7l909yeSr89JelrSWCtLRehGSafc/Ul3/4akuxTOYVb2nN4j6XXJtXaLpLvc/Vl3/xtJp5Lv1xYHnjt3/2Tm79qDkq4uuY11Ns61N8qbJN3n7s+4+9ck3Sfpphm1s44mPXdvlfShUlrWAO7+5wodeaPcIukPPHhQUsfMrlJLrztCdfVe7O5PSVLy+UUHHH9EFz/hV5O3XU6Y2WWzaGRNjXvuXmBmm2b2YFo2I+mFkrbd/blk+6ykl862ubUz0bVnZjcq9PR8KbO7TdfeSyV9ObM97Jr55jHJtfV1hWttnPvGbNLH/w6F3q/UsOdwm4x7/n4seT7eY2bXTHjfWI39+JOSo2slPZDZ3fZr7yCjzm8rr7tDVTegDczsE5JeMuSmlQm/z1WSXi7p3szu90r6O4Wwsy7pPZJun66l9VPQuVtw93Nmdp2kB8zsUUn/MOS46OaXLPja+0NJR919J9kd9bU3hA3ZN3jNjDpmnPvGbOzHb2Zvk7Qs6VWZ3Rc9h939S8PuH6lxzt+fSPqQuz9rZu9SeMfktWPeN2aTPP4jku5x9+cz+9p+7R2Ev3kZhOoSuPvrR91mZl8xs6vc/akkuDy9z7d6i6Q/dvd/yXzvp5IvnzWz35f0i4U0uiaKOHdJ2YLc/Ukz+5SkV0j63wpvUx1KehSvlnSu8AdQsSLOn5l9q6Q/k/RLydt76feO+tob4qykazLbw66Z9JizZnZI0rcpvHU6zn1jNtbjN7PXK/zD9yp3fzbdP+I53KZgc+D5c/evZjZ/V9KvZu776oH7fqrwFtbXJM+9I5J+LruDa+9Ao85vK687yj+qd1JSOir2qKSP7nPsRbVeSRhKa4RvlTR0hG6kDjx3ZnY4LUswsyskvVLSYx5GUnxSoUZ95P0jN875u1TSHyvUzH1k4La2XXsPSbrewqwxlyq8AA/OBpA9p2+W9EByrZ2UdMTC7CDXSrpe0l+V1O46OPDcmdkrJP2OpJvd/enM/qHP4dJaXg/jnL+rMps3S/pC8vW9kt6YnMfDkt6ove92xm6c563M7DsVBtT9ZWYf197BTkr6yWQWkB+S9PWkw6Wd113VIyXb/qFQb3m/pCeSz5cn+5clvT9z3JKkv5U0N3D/ByQ9qhBo/kjSt1T9mOp07iT9cHJ+PpN8fkfm/tcpBJtTkj4i6bKqH1MNz9/bJP2LpEcyHze09dpTGOn+RYWeqpVk3+0KQVCSXpBcS6eSa+u6zH1Xkvs9LulHq34sNTx3n5D0lcx1djLZP/I53KaPMc7ff5f0+eQ8fVLSd2Xu+9PJNXlK0k9V/Vjqdu6S7b6kOwbu1/prT6Ej76nkdeCswniHd0l6V3K7SfrN5Nw+Kmk5c9/WXXcsUw4AAADkRPkHAAAAkBOhGgAAAMiJUA0AAADkRKgGAAAAciJUAwAAADkRqgEAAICcCNUAAABATv8fA39wCzao42UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(12,8))\n",
    "ax.scatter(positive[\"Test1\"], positive[\"Test2\"], c = 'r', marker = 'o')\n",
    "ax.scatter(negative[\"Test1\"], negative[\"Test2\"], c = 'b', marker = 'x')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Feature Mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Test1</th>\n",
       "      <th>Test2</th>\n",
       "      <th>Result</th>\n",
       "      <th>F10</th>\n",
       "      <th>F01</th>\n",
       "      <th>F20</th>\n",
       "      <th>F11</th>\n",
       "      <th>F02</th>\n",
       "      <th>F30</th>\n",
       "      <th>F21</th>\n",
       "      <th>...</th>\n",
       "      <th>F23</th>\n",
       "      <th>F14</th>\n",
       "      <th>F05</th>\n",
       "      <th>F60</th>\n",
       "      <th>F51</th>\n",
       "      <th>F42</th>\n",
       "      <th>F33</th>\n",
       "      <th>F24</th>\n",
       "      <th>F15</th>\n",
       "      <th>F06</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.051267</td>\n",
       "      <td>0.69956</td>\n",
       "      <td>1</td>\n",
       "      <td>0.051267</td>\n",
       "      <td>0.69956</td>\n",
       "      <td>0.002628</td>\n",
       "      <td>0.035864</td>\n",
       "      <td>0.489384</td>\n",
       "      <td>0.000135</td>\n",
       "      <td>0.001839</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000900</td>\n",
       "      <td>0.012278</td>\n",
       "      <td>0.167542</td>\n",
       "      <td>1.815630e-08</td>\n",
       "      <td>2.477505e-07</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>0.000046</td>\n",
       "      <td>0.000629</td>\n",
       "      <td>0.008589</td>\n",
       "      <td>0.117206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.092742</td>\n",
       "      <td>0.68494</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.092742</td>\n",
       "      <td>0.68494</td>\n",
       "      <td>0.008601</td>\n",
       "      <td>-0.063523</td>\n",
       "      <td>0.469143</td>\n",
       "      <td>-0.000798</td>\n",
       "      <td>0.005891</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002764</td>\n",
       "      <td>-0.020412</td>\n",
       "      <td>0.150752</td>\n",
       "      <td>6.362953e-07</td>\n",
       "      <td>-4.699318e-06</td>\n",
       "      <td>0.000035</td>\n",
       "      <td>-0.000256</td>\n",
       "      <td>0.001893</td>\n",
       "      <td>-0.013981</td>\n",
       "      <td>0.103256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.213710</td>\n",
       "      <td>0.69225</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.213710</td>\n",
       "      <td>0.69225</td>\n",
       "      <td>0.045672</td>\n",
       "      <td>-0.147941</td>\n",
       "      <td>0.479210</td>\n",
       "      <td>-0.009761</td>\n",
       "      <td>0.031616</td>\n",
       "      <td>...</td>\n",
       "      <td>0.015151</td>\n",
       "      <td>-0.049077</td>\n",
       "      <td>0.158970</td>\n",
       "      <td>9.526844e-05</td>\n",
       "      <td>-3.085938e-04</td>\n",
       "      <td>0.001000</td>\n",
       "      <td>-0.003238</td>\n",
       "      <td>0.010488</td>\n",
       "      <td>-0.033973</td>\n",
       "      <td>0.110047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.375000</td>\n",
       "      <td>0.50219</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.375000</td>\n",
       "      <td>0.50219</td>\n",
       "      <td>0.140625</td>\n",
       "      <td>-0.188321</td>\n",
       "      <td>0.252195</td>\n",
       "      <td>-0.052734</td>\n",
       "      <td>0.070620</td>\n",
       "      <td>...</td>\n",
       "      <td>0.017810</td>\n",
       "      <td>-0.023851</td>\n",
       "      <td>0.031940</td>\n",
       "      <td>2.780914e-03</td>\n",
       "      <td>-3.724126e-03</td>\n",
       "      <td>0.004987</td>\n",
       "      <td>-0.006679</td>\n",
       "      <td>0.008944</td>\n",
       "      <td>-0.011978</td>\n",
       "      <td>0.016040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.513250</td>\n",
       "      <td>0.46564</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.513250</td>\n",
       "      <td>0.46564</td>\n",
       "      <td>0.263426</td>\n",
       "      <td>-0.238990</td>\n",
       "      <td>0.216821</td>\n",
       "      <td>-0.135203</td>\n",
       "      <td>0.122661</td>\n",
       "      <td>...</td>\n",
       "      <td>0.026596</td>\n",
       "      <td>-0.024128</td>\n",
       "      <td>0.021890</td>\n",
       "      <td>1.827990e-02</td>\n",
       "      <td>-1.658422e-02</td>\n",
       "      <td>0.015046</td>\n",
       "      <td>-0.013650</td>\n",
       "      <td>0.012384</td>\n",
       "      <td>-0.011235</td>\n",
       "      <td>0.010193</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Test1    Test2  Result       F10      F01       F20       F11       F02  \\\n",
       "0  0.051267  0.69956       1  0.051267  0.69956  0.002628  0.035864  0.489384   \n",
       "1 -0.092742  0.68494       1 -0.092742  0.68494  0.008601 -0.063523  0.469143   \n",
       "2 -0.213710  0.69225       1 -0.213710  0.69225  0.045672 -0.147941  0.479210   \n",
       "3 -0.375000  0.50219       1 -0.375000  0.50219  0.140625 -0.188321  0.252195   \n",
       "4 -0.513250  0.46564       1 -0.513250  0.46564  0.263426 -0.238990  0.216821   \n",
       "\n",
       "        F30       F21  ...       F23       F14       F05           F60  \\\n",
       "0  0.000135  0.001839  ...  0.000900  0.012278  0.167542  1.815630e-08   \n",
       "1 -0.000798  0.005891  ...  0.002764 -0.020412  0.150752  6.362953e-07   \n",
       "2 -0.009761  0.031616  ...  0.015151 -0.049077  0.158970  9.526844e-05   \n",
       "3 -0.052734  0.070620  ...  0.017810 -0.023851  0.031940  2.780914e-03   \n",
       "4 -0.135203  0.122661  ...  0.026596 -0.024128  0.021890  1.827990e-02   \n",
       "\n",
       "            F51       F42       F33       F24       F15       F06  \n",
       "0  2.477505e-07  0.000003  0.000046  0.000629  0.008589  0.117206  \n",
       "1 -4.699318e-06  0.000035 -0.000256  0.001893 -0.013981  0.103256  \n",
       "2 -3.085938e-04  0.001000 -0.003238  0.010488 -0.033973  0.110047  \n",
       "3 -3.724126e-03  0.004987 -0.006679  0.008944 -0.011978  0.016040  \n",
       "4 -1.658422e-02  0.015046 -0.013650  0.012384 -0.011235  0.010193  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(1,7):\n",
    "    for j in range(0,i+1):\n",
    "        s = 'F'+str(i-j)+str(j)\n",
    "        data[s] = np.power(data[\"Test1\"], i-j)*np.power(data[\"Test2\"], j)\n",
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Result</th>\n",
       "      <th>F10</th>\n",
       "      <th>F01</th>\n",
       "      <th>F20</th>\n",
       "      <th>F11</th>\n",
       "      <th>F02</th>\n",
       "      <th>F30</th>\n",
       "      <th>F21</th>\n",
       "      <th>F12</th>\n",
       "      <th>F03</th>\n",
       "      <th>...</th>\n",
       "      <th>F23</th>\n",
       "      <th>F14</th>\n",
       "      <th>F05</th>\n",
       "      <th>F60</th>\n",
       "      <th>F51</th>\n",
       "      <th>F42</th>\n",
       "      <th>F33</th>\n",
       "      <th>F24</th>\n",
       "      <th>F15</th>\n",
       "      <th>F06</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.051267</td>\n",
       "      <td>0.69956</td>\n",
       "      <td>0.002628</td>\n",
       "      <td>0.035864</td>\n",
       "      <td>0.489384</td>\n",
       "      <td>0.000135</td>\n",
       "      <td>0.001839</td>\n",
       "      <td>0.025089</td>\n",
       "      <td>0.342354</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000900</td>\n",
       "      <td>0.012278</td>\n",
       "      <td>0.167542</td>\n",
       "      <td>1.815630e-08</td>\n",
       "      <td>2.477505e-07</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>0.000046</td>\n",
       "      <td>0.000629</td>\n",
       "      <td>0.008589</td>\n",
       "      <td>0.117206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.092742</td>\n",
       "      <td>0.68494</td>\n",
       "      <td>0.008601</td>\n",
       "      <td>-0.063523</td>\n",
       "      <td>0.469143</td>\n",
       "      <td>-0.000798</td>\n",
       "      <td>0.005891</td>\n",
       "      <td>-0.043509</td>\n",
       "      <td>0.321335</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002764</td>\n",
       "      <td>-0.020412</td>\n",
       "      <td>0.150752</td>\n",
       "      <td>6.362953e-07</td>\n",
       "      <td>-4.699318e-06</td>\n",
       "      <td>0.000035</td>\n",
       "      <td>-0.000256</td>\n",
       "      <td>0.001893</td>\n",
       "      <td>-0.013981</td>\n",
       "      <td>0.103256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.213710</td>\n",
       "      <td>0.69225</td>\n",
       "      <td>0.045672</td>\n",
       "      <td>-0.147941</td>\n",
       "      <td>0.479210</td>\n",
       "      <td>-0.009761</td>\n",
       "      <td>0.031616</td>\n",
       "      <td>-0.102412</td>\n",
       "      <td>0.331733</td>\n",
       "      <td>...</td>\n",
       "      <td>0.015151</td>\n",
       "      <td>-0.049077</td>\n",
       "      <td>0.158970</td>\n",
       "      <td>9.526844e-05</td>\n",
       "      <td>-3.085938e-04</td>\n",
       "      <td>0.001000</td>\n",
       "      <td>-0.003238</td>\n",
       "      <td>0.010488</td>\n",
       "      <td>-0.033973</td>\n",
       "      <td>0.110047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.375000</td>\n",
       "      <td>0.50219</td>\n",
       "      <td>0.140625</td>\n",
       "      <td>-0.188321</td>\n",
       "      <td>0.252195</td>\n",
       "      <td>-0.052734</td>\n",
       "      <td>0.070620</td>\n",
       "      <td>-0.094573</td>\n",
       "      <td>0.126650</td>\n",
       "      <td>...</td>\n",
       "      <td>0.017810</td>\n",
       "      <td>-0.023851</td>\n",
       "      <td>0.031940</td>\n",
       "      <td>2.780914e-03</td>\n",
       "      <td>-3.724126e-03</td>\n",
       "      <td>0.004987</td>\n",
       "      <td>-0.006679</td>\n",
       "      <td>0.008944</td>\n",
       "      <td>-0.011978</td>\n",
       "      <td>0.016040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.513250</td>\n",
       "      <td>0.46564</td>\n",
       "      <td>0.263426</td>\n",
       "      <td>-0.238990</td>\n",
       "      <td>0.216821</td>\n",
       "      <td>-0.135203</td>\n",
       "      <td>0.122661</td>\n",
       "      <td>-0.111283</td>\n",
       "      <td>0.100960</td>\n",
       "      <td>...</td>\n",
       "      <td>0.026596</td>\n",
       "      <td>-0.024128</td>\n",
       "      <td>0.021890</td>\n",
       "      <td>1.827990e-02</td>\n",
       "      <td>-1.658422e-02</td>\n",
       "      <td>0.015046</td>\n",
       "      <td>-0.013650</td>\n",
       "      <td>0.012384</td>\n",
       "      <td>-0.011235</td>\n",
       "      <td>0.010193</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Result       F10      F01       F20       F11       F02       F30  \\\n",
       "0       1  0.051267  0.69956  0.002628  0.035864  0.489384  0.000135   \n",
       "1       1 -0.092742  0.68494  0.008601 -0.063523  0.469143 -0.000798   \n",
       "2       1 -0.213710  0.69225  0.045672 -0.147941  0.479210 -0.009761   \n",
       "3       1 -0.375000  0.50219  0.140625 -0.188321  0.252195 -0.052734   \n",
       "4       1 -0.513250  0.46564  0.263426 -0.238990  0.216821 -0.135203   \n",
       "\n",
       "        F21       F12       F03  ...       F23       F14       F05  \\\n",
       "0  0.001839  0.025089  0.342354  ...  0.000900  0.012278  0.167542   \n",
       "1  0.005891 -0.043509  0.321335  ...  0.002764 -0.020412  0.150752   \n",
       "2  0.031616 -0.102412  0.331733  ...  0.015151 -0.049077  0.158970   \n",
       "3  0.070620 -0.094573  0.126650  ...  0.017810 -0.023851  0.031940   \n",
       "4  0.122661 -0.111283  0.100960  ...  0.026596 -0.024128  0.021890   \n",
       "\n",
       "            F60           F51       F42       F33       F24       F15  \\\n",
       "0  1.815630e-08  2.477505e-07  0.000003  0.000046  0.000629  0.008589   \n",
       "1  6.362953e-07 -4.699318e-06  0.000035 -0.000256  0.001893 -0.013981   \n",
       "2  9.526844e-05 -3.085938e-04  0.001000 -0.003238  0.010488 -0.033973   \n",
       "3  2.780914e-03 -3.724126e-03  0.004987 -0.006679  0.008944 -0.011978   \n",
       "4  1.827990e-02 -1.658422e-02  0.015046 -0.013650  0.012384 -0.011235   \n",
       "\n",
       "        F06  \n",
       "0  0.117206  \n",
       "1  0.103256  \n",
       "2  0.110047  \n",
       "3  0.016040  \n",
       "4  0.010193  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data.drop(columns=[\"Test1\", \"Test2\"])\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Result</th>\n",
       "      <th>F00</th>\n",
       "      <th>F10</th>\n",
       "      <th>F01</th>\n",
       "      <th>F20</th>\n",
       "      <th>F11</th>\n",
       "      <th>F02</th>\n",
       "      <th>F30</th>\n",
       "      <th>F21</th>\n",
       "      <th>F12</th>\n",
       "      <th>...</th>\n",
       "      <th>F23</th>\n",
       "      <th>F14</th>\n",
       "      <th>F05</th>\n",
       "      <th>F60</th>\n",
       "      <th>F51</th>\n",
       "      <th>F42</th>\n",
       "      <th>F33</th>\n",
       "      <th>F24</th>\n",
       "      <th>F15</th>\n",
       "      <th>F06</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.051267</td>\n",
       "      <td>0.69956</td>\n",
       "      <td>0.002628</td>\n",
       "      <td>0.035864</td>\n",
       "      <td>0.489384</td>\n",
       "      <td>0.000135</td>\n",
       "      <td>0.001839</td>\n",
       "      <td>0.025089</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000900</td>\n",
       "      <td>0.012278</td>\n",
       "      <td>0.167542</td>\n",
       "      <td>1.815630e-08</td>\n",
       "      <td>2.477505e-07</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>0.000046</td>\n",
       "      <td>0.000629</td>\n",
       "      <td>0.008589</td>\n",
       "      <td>0.117206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.092742</td>\n",
       "      <td>0.68494</td>\n",
       "      <td>0.008601</td>\n",
       "      <td>-0.063523</td>\n",
       "      <td>0.469143</td>\n",
       "      <td>-0.000798</td>\n",
       "      <td>0.005891</td>\n",
       "      <td>-0.043509</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002764</td>\n",
       "      <td>-0.020412</td>\n",
       "      <td>0.150752</td>\n",
       "      <td>6.362953e-07</td>\n",
       "      <td>-4.699318e-06</td>\n",
       "      <td>0.000035</td>\n",
       "      <td>-0.000256</td>\n",
       "      <td>0.001893</td>\n",
       "      <td>-0.013981</td>\n",
       "      <td>0.103256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.213710</td>\n",
       "      <td>0.69225</td>\n",
       "      <td>0.045672</td>\n",
       "      <td>-0.147941</td>\n",
       "      <td>0.479210</td>\n",
       "      <td>-0.009761</td>\n",
       "      <td>0.031616</td>\n",
       "      <td>-0.102412</td>\n",
       "      <td>...</td>\n",
       "      <td>0.015151</td>\n",
       "      <td>-0.049077</td>\n",
       "      <td>0.158970</td>\n",
       "      <td>9.526844e-05</td>\n",
       "      <td>-3.085938e-04</td>\n",
       "      <td>0.001000</td>\n",
       "      <td>-0.003238</td>\n",
       "      <td>0.010488</td>\n",
       "      <td>-0.033973</td>\n",
       "      <td>0.110047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.375000</td>\n",
       "      <td>0.50219</td>\n",
       "      <td>0.140625</td>\n",
       "      <td>-0.188321</td>\n",
       "      <td>0.252195</td>\n",
       "      <td>-0.052734</td>\n",
       "      <td>0.070620</td>\n",
       "      <td>-0.094573</td>\n",
       "      <td>...</td>\n",
       "      <td>0.017810</td>\n",
       "      <td>-0.023851</td>\n",
       "      <td>0.031940</td>\n",
       "      <td>2.780914e-03</td>\n",
       "      <td>-3.724126e-03</td>\n",
       "      <td>0.004987</td>\n",
       "      <td>-0.006679</td>\n",
       "      <td>0.008944</td>\n",
       "      <td>-0.011978</td>\n",
       "      <td>0.016040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.513250</td>\n",
       "      <td>0.46564</td>\n",
       "      <td>0.263426</td>\n",
       "      <td>-0.238990</td>\n",
       "      <td>0.216821</td>\n",
       "      <td>-0.135203</td>\n",
       "      <td>0.122661</td>\n",
       "      <td>-0.111283</td>\n",
       "      <td>...</td>\n",
       "      <td>0.026596</td>\n",
       "      <td>-0.024128</td>\n",
       "      <td>0.021890</td>\n",
       "      <td>1.827990e-02</td>\n",
       "      <td>-1.658422e-02</td>\n",
       "      <td>0.015046</td>\n",
       "      <td>-0.013650</td>\n",
       "      <td>0.012384</td>\n",
       "      <td>-0.011235</td>\n",
       "      <td>0.010193</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Result  F00       F10      F01       F20       F11       F02       F30  \\\n",
       "0       1    1  0.051267  0.69956  0.002628  0.035864  0.489384  0.000135   \n",
       "1       1    1 -0.092742  0.68494  0.008601 -0.063523  0.469143 -0.000798   \n",
       "2       1    1 -0.213710  0.69225  0.045672 -0.147941  0.479210 -0.009761   \n",
       "3       1    1 -0.375000  0.50219  0.140625 -0.188321  0.252195 -0.052734   \n",
       "4       1    1 -0.513250  0.46564  0.263426 -0.238990  0.216821 -0.135203   \n",
       "\n",
       "        F21       F12  ...       F23       F14       F05           F60  \\\n",
       "0  0.001839  0.025089  ...  0.000900  0.012278  0.167542  1.815630e-08   \n",
       "1  0.005891 -0.043509  ...  0.002764 -0.020412  0.150752  6.362953e-07   \n",
       "2  0.031616 -0.102412  ...  0.015151 -0.049077  0.158970  9.526844e-05   \n",
       "3  0.070620 -0.094573  ...  0.017810 -0.023851  0.031940  2.780914e-03   \n",
       "4  0.122661 -0.111283  ...  0.026596 -0.024128  0.021890  1.827990e-02   \n",
       "\n",
       "            F51       F42       F33       F24       F15       F06  \n",
       "0  2.477505e-07  0.000003  0.000046  0.000629  0.008589  0.117206  \n",
       "1 -4.699318e-06  0.000035 -0.000256  0.001893 -0.013981  0.103256  \n",
       "2 -3.085938e-04  0.001000 -0.003238  0.010488 -0.033973  0.110047  \n",
       "3 -3.724126e-03  0.004987 -0.006679  0.008944 -0.011978  0.016040  \n",
       "4 -1.658422e-02  0.015046 -0.013650  0.012384 -0.011235  0.010193  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.insert(1, \"F00\", 1)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(118, 1)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = data[\"Result\"].values\n",
    "y = np.expand_dims(y, axis = 1)\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(118, 28)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = data.iloc[:,1:29].values\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28,)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta = np.zeros(28)\n",
    "theta.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAG7pJREFUeJzt3XuUVOWd7vHvj+ZmIghycRCQixAdjM6oLYqaqNExDUGIDnLJGM2JS5I45GRW4hlRDBqcyeiMnjPJEi+YMBqSAvGOph3ijbhMRAEjCCrS4oUeEPAK3oCW3/njrZairaZ3d1fVW5fns1atrqq9u/phd/GweWvv/Zq7IyIi5aVD7AAiIpJ7KncRkTKkchcRKUMqdxGRMqRyFxEpQyp3EZEypHIXESlDKncRkTKkchcRKUMdY/3g3r17++DBg2P9eBGRkrRixYq33L1PS+tFK/fBgwezfPnyWD9eRKQkmdnrSdbTsIyISBlSuYuIlCGVu4hIGVK5i4iUIZW7iEgZarHczWyumW0xs9XNLDcz+6WZ1ZnZKjM7JvcxRUSkNZLsud8G1Oxj+WhgePo2Fbip/bFERKQ9WjzO3d2fMLPB+1hlPPAbD/P1LTWzHmbWz9035SijiEjgDh9/DDt2QEMD7NrVulvj9+zevffNPdlzjc9n3hpzteb+WWfBccfldVPl4iSm/sCGjMf16ec+V+5mNpWwd88hhxySgx8tIiXl3Xfh1Vfhvfdg+3bYtm3vry09t317KNhSZgYHH1wS5W5Znss667a7zwHmAFRXV2tmbpFy4w5btkBdXbi98sre9995p/nvraqC7t2hW7dw694dDjgABgzY+/n994cuXaBTp9bfOnYMX6uqoEOHcDPbc39fzzU+37gM9jxOer+AclHu9cDAjMcDgI05eF0RKUa7d0N9/d7lnVniH364Z90OHWDQIBg2DCZNgkMPhaFD4cAD9xR249euXaOUYLnKRbkvAqaZ2QLgeOB9jbeLlJmGBnj0UZg3D+67b+8C79wZhgwJBX7qqaHAhw0Lt0GDwnIpuBbL3czmA6cCvc2sHrgS6ATg7jcDtcAYoA74CPhf+QorIgXkDitXhkJPpeDNN6FHD/jWt+DYY/cU+IABYZhDikqSo2WmtLDcgX/MWSIRiau+PpT5vHmwenUYo/7GN+Db3w5fu3SJnVASiHbJXxEpItu3wz33hEJ/7LGw1z5qFNx4I0ycCL16xU4oraRyF6lUDQ3wyCOh0O+9Nxw/PnQozJwJ550XhlykZKncRSrNypVw++1h6GXzZujZEy64IAy7jBqlI1bKhMpdpFK4w7/8S9gz79QJxo4NhT5mjMbRy5DKXaQSNDTAxRfDrbeGQv/P/wzHmkvZUrmLlLsPPwwnEP3+93D55WHvXUMvZU/lLlLOtmwJwy8rVsBNN8H3vx87kRSIyl2kXNXVQU0NbNwYjoYZNy52IikglbtIOXr66bDHDuG49RNOiJtHCk7T7ImUmwcegNNOCxfk+vOfVewVSuUuUk5uuQW++U044ohQ7MOHx04kkajcRcqBO1xxRfjAtKYGHn8cDjoodiqJSGPuIqVu1y646KJw1umFF8LNN4dJKaSiac9dpJRt3x4+OL39drjqqnCSkopd0J67SOnatClcgnfVKvj1r+G7342dSIqIyl2kFL30Uhhbf+utcHTM6NGxE0mRUbmLlJo//SmckNSxIyxZAtXVsRNJEdKYu0gpWbUKzjgjTJ7x1FMqdmmW9txFSsk//zPstx88+ST07Rs7jRQxlbtIqXj0UVi8GK67TsUuLdKwjEgp2L0bLr0UDjkE/lHz0UvLtOcuUgoWLgyX7b39dujaNXYaKQHacxcpdjt3wowZcOSR8A//EDuNlAjtuYsUu1tugfXrobYWqqpip5ESoT13kWK2bRvMmhUu4VtTEzuNlBCVu0gxu+66cBbqtddq3lNpFZW7SLF68024/nqYOBGOOy52GikxKneRYvWzn4UPU//1X2MnkRKkchcpRi+/HC7f+73vwbBhsdNICVK5ixSjyy8Plxn46U9jJ5ESlajczazGzNaaWZ2ZTc+y/BAze9zM/mJmq8xsTO6jilSIpUvh7rvhkks0VZ60WYvlbmZVwGxgNDACmGJmI5qsdgWw0N2PBiYDN+Y6qEhFcA+XGejbF37849hppIQlOYlpJFDn7usBzGwBMB54IWMdB7qn7x8AbMxlSJGKUVsLTzwBs2dDt26x00gJS1Lu/YENGY/rgeObrHMV8Acz+yHwReCMnKQTqSSffgrTp4cPUC+6KHYaKXFJxtyznTnhTR5PAW5z9wHAGGCemX3utc1sqpktN7PlW7dubX1akXI2bx6sXg0//zl06hQ7jZS4JOVeDwzMeDyAzw+7XAgsBHD3p4CuQO+mL+Tuc9y92t2r+/Tp07bEIuXo44/DkTEjR8KECbHTSBlIUu7LgOFmNsTMOhM+MF3UZJ03gNMBzOyvCeWuXXORpG64AerrdZkByZkWy93dG4BpwGLgRcJRMWvMbJaZjUuv9hPgIjNbCcwHvuPuTYduRCSbd94JQzFjxsCpp8ZOI2Ui0SV/3b0WqG3y3MyM+y8AJ+U2mkiFuOYaeP99+Ld/i51EyojOUBWJ6Y034Je/hPPPh6OOip1GyojKXSSmK68MX2fNiptDyo7KXSSW558Pc6L+8Idh4muRHFK5i8QyfToccABcdlnsJFKGNIeqSAxLloRLDVx7LRx4YOw0Uoa05y5SaI0XBxswIAzJiOSB9txFCu3uu+GZZ2Du3HDNdpE80J67SCHt2hUm4jjiiHD4o0ieaM9dpJDuuw/WrQtfq6pip5Eypj13kUJKpaBfPxg7NnYSKXMqd5FCee+9cITMpEnaa5e8U7mLFMo998DOnfCtb8VOIhVA5S5SKKlUmGWpujp2EqkAKneRQti0CR57LOy163rtUgAqd5FCWLgwnLw0ZUrsJFIhVO4ihZBKwdFHw+GHx04iFULlLpJvdXXhjFR9kCoFpHIXybf588M4++TJsZNIBVG5i+STexiS+epXw4XCRApE5S6STytXwksv6YNUKTiVu0g+pVLQsSNMmBA7iVQYlbtIvuzeHcbba2qgV6/YaaTCqNxF8uXJJ6G+XkfJSBQqd5F8mT8fvvAFGDcudhKpQCp3kXzYuTOclTp+PHzxi7HTSAVSuYvkw8MPwzvvaEhGolG5i+RDKgUHHghnnhk7iVQolbtIrn34Idx/P5x7LnTuHDuNVCiVu0iuPfBAKHiduCQRqdxFci2Vgv794StfiZ1EKpjKXSSX3n4bHnoo7LV30F8viSfRu8/MasxsrZnVmdn0ZtaZaGYvmNkaM0vlNqZIibj7bmho0FEyEl3HllYwsypgNvB3QD2wzMwWufsLGesMBy4DTnL3d82sb74CixS1+fPhsMPgb/82dhKpcEn23EcCde6+3t13AguA8U3WuQiY7e7vArj7ltzGFCkB9fXwxz9qnlQpCknKvT+wIeNxffq5TF8CvmRmfzKzpWZWk+2FzGyqmS03s+Vbt25tW2KRYnXHHZonVYpGknLPtgviTR53BIYDpwJTgF+ZWY/PfZP7HHevdvfqPn36tDarSHFLpeC442D48NhJRBKVez0wMOPxAGBjlnXud/dd7v4qsJZQ9iKVYe1aePZZfZAqRSNJuS8DhpvZEDPrDEwGFjVZ5z7gNAAz600Yplmfy6AiRa1xntSJE2MnEQESlLu7NwDTgMXAi8BCd19jZrPMrPFapouBt83sBeBx4P+4+9v5Ci1SVBrnST3tNDj44NhpRIAEh0ICuHstUNvkuZkZ9x34cfomUllWrIB16+DSS2MnEfmMTqETaa9UKlwg7JxzYicR+YzKXaQ9Pv00HAI5ejT07Bk7jchnVO4i7fHEE7Bxo46SkaKjchdpj1QK9t8fxo6NnURkLyp3kbbasQPuugvOPjtMhC1SRFTuIm21eDG8956GZKQoqdxF2iqVgt694fTTYycR+RyVu0hbbN8OixaFM1I7dYqdRuRzVO4ibXH//fDxxxqSkaKlchdpi/nzYdAgGDUqdhKRrFTuIq21dWv4MHXyZM2TKkVL70yR1rrrrnBmqoZkpIip3EVaK5WCI46AI4+MnUSkWSp3kdZ44w148knNkypFT+Uu0hoLFoSvkyfHzSHSApW7SGukUnDCCTB0aOwkIvukchdJas0aWLlSH6RKSVC5iyQ1f3449FHzpEoJULmLJOEeyv2MM+Cgg2KnEWmRyl0kiWeegfXrYcqU2ElEElG5iySRSkGXLuHa7SIlQOUu0pKGhjBP6tixcMABsdOIJKJyF2nJkiWwebOOkpGSonIXaUkqBd27w5gxsZOIJKZyF9mXTz6Bu++Gc86Brl1jpxFJTOUusi+1tbBtm4ZkpOSo3EX2JZUKx7WfdlrsJCKtonIXac62bfDggzBpEnTsGDuNSKuo3EWac++9sGOHTlySkqRyF2lOKgVDhsDxx8dOItJqicrdzGrMbK2Z1ZnZ9H2sN8HM3MyqcxdRJILNm+GRRzQph5SsFsvdzKqA2cBoYAQwxcxGZFmvG/C/gadzHVKk4O68E3bv1lEyUrKS7LmPBOrcfb277wQWAOOzrHc18O/AJznMJxJHKgVHHQUjPrcfI1ISkpR7f2BDxuP69HOfMbOjgYHu/mAOs4nEsX49PPWU9tqlpCUp92wDjv7ZQrMOwP8DftLiC5lNNbPlZrZ869atyVOKFJLmSZUykKTc64GBGY8HABszHncDvgwsMbPXgBOARdk+VHX3Oe5e7e7Vffr0aXtqkXxKpeDkk2HQoNhJRNosSbkvA4ab2RAz6wxMBhY1LnT39929t7sPdvfBwFJgnLsvz0tikXx6/vkwV6qObZcS12K5u3sDMA1YDLwILHT3NWY2y8zG5TugSEGlUlBVBeeeGzuJSLskOqfa3WuB2ibPzWxm3VPbH0skgt27wzypZ54JGjaUEqczVEUaPfUUvP66jpKRsqByF2k0fz7stx+Mz3Yah0hpUbmLAOzaBQsXwllnQbdusdOItJvKXQTg0Udh61YNyUjZULmLQDhKpkcPqKmJnUQkJ1TuIh99FK7dPmECdOkSO41ITqjcRX7/e/jgA524JGVF5S6SSkG/fnDKKbGTiOSMyl0q27vvQm1tuEhYVVXsNCI5o3KXynbPPbBzp46SkbKjcpfKNn8+DBsGxx4bO4lITqncpXJt2gSPPaZ5UqUsqdylct1xB7jrKBkpSyp3qVypFBxzDBx+eOwkIjmncpfKtG4dLFumD1KlbKncpTItWBDG2SdNip1EJC9U7lJ53OF3v4OvfhUGDIidRiQvVO5SeZ57Dtau1ZCMlDWVu1SeVAo6dYK///vYSUTyRuUulWX37jDe/vWvQ69esdOI5I3KXSrLI49Afb2GZKTsqdylcrjDjBkwcCCcfXbsNCJ51TF2AJGCufNOWL4cbrsNunaNnUYkr7TnLpVh1y64/HI48kg477zYaUTyTnvuUhnmzIFXXgmzLum67VIBtOcu5W/7dpg1K8y0NHp07DQiBaE9dyl/118PW7bAAw/o0r5SMbTnLuVt82a47jqYMAFGjoydRqRgVO5S3mbNgh074Oc/j51EpKBU7lK+1q0LH6ROnQrDh8dOI1JQKncpXzNmQJcuMHNm7CQiBZeo3M2sxszWmlmdmU3PsvzHZvaCma0ys0fNbFDuo4q0wjPPhJOWLrkEDjoodhqRgmux3M2sCpgNjAZGAFPMbEST1f4CVLv7UcBdwL/nOqhIYu5w6aXQty/85Cex04hEkWTPfSRQ5+7r3X0nsAAYn7mCuz/u7h+lHy4FNAOCxPPf/w1LloThmG7dYqcRiSJJufcHNmQ8rk8/15wLgYeyLTCzqWa23MyWb926NXlKkaQ+/TTstR96KFx0Uew0ItEkOYkp21kfnnVFs/OAauCUbMvdfQ4wB6C6ujrra4i0y29/C88/D3fcAZ07x04jEk2Scq8HBmY8HgBsbLqSmZ0BzABOcfcduYkn0gqffAI//SlUV4eTlkQqWJJyXwYMN7MhwP8Ak4G9Zjows6OBW4Aad9+S85QiSdxwA2zYALffDh10lK9Uthb/Brh7AzANWAy8CCx09zVmNsvMxqVX+w9gf+BOM3vOzBblLbFINu++G85CramB006LnUYkukQXDnP3WqC2yXMzM+6fkeNcIq1zzTXw3nvhq4joDFUpAxs2wC9+ESbh+Ju/iZ1GpCio3KX0XXllOHHp6qtjJxEpGip3KW2rV4cPUKdNg0G66oVII5W7lLbLLgtnoV5+eewkIkVF5S6l64kn4MEHYfp06NUrdhqRoqJyl9LUeHGw/v3hRz+KnUak6GgOVSlN994LS5fCr34F++0XO41I0dGeu5SeXbvCWPuIEXDBBbHTiBQl7blL6Zk7F15+Ge6/HzrqLSySjfbcpbR88AFcdRWcfDKcdVbsNCJFS7s9UjreegvGjYPNm+Gee8CyXY1aREDlLqVi/XoYPRpefz3MjTpqVOxEIkVN5S7Fb8UKGDMmfJD66KNw0kmxE4kUPY25S3F76CE45ZRwuOOf/6xiF0lI5S7Fa+7c8KHpl74ETz0Fhx8eO5FIyVC5S/Fxh5/9DC68EE4/Hf74R+jXL3YqkZKiMXcpLg0N8IMfhDNPL7gAbr0VOnWKnUqk5GjPXYrHBx/A+PGh2K+4Av7rv1TsIm2kPXcpDps3w9ix8OyzcPPN8L3vxU4kUtJU7hLfyy+HY9g3bYL77tOZpyI5oHKXuJYuDXvsZrBkCYwcGTuRSFnQmLvEs2gRfO1r0KNHONRRxS6SMyp3ieOmm+Dss+HLXw4nJw0bFjuRSFlRuUthNTSE+U4vvjhcUuDxx6Fv39ipRMqOxtwl/9zhuedg3jyYPx/efBOmToXZs3U9dpE80d8syZ/6ekilQqmvXh2OWf/GN+A73wmX7tUle0XyRuUuubV9e7jW+rx58NhjYa991Ci48UaYOBF69YqdUKQiqNyl/Roa4OGH4be/DRNXf/wxDB0KM2fCeefpw1KRCFTu0jbu8Je/7BlH37wZevYM14P59rfD3rqGXUSiUblLMp9+Chs2QF0dLFsW9tJfeCGMo48dGwp9zBjo0iV2UhEhYbmbWQ3wC6AK+JW7X9NkeRfgN8CxwNvAJHd/LbdRJe927oTXXgsFXlcHr7yy5/6rr4aZkBqdeGI4Vn3iRDjwwGiRRSS7FsvdzKqA2cDfAfXAMjNb5O4vZKx2IfCuuw8zs8nAtcCkfASWNtq9O1x1cft2eOedPcWdWeBvvBHWa7T//mG8/MgjwwlHw4bBoYfCYYfBwQfH+7OISIuS7LmPBOrcfT2AmS0AxgOZ5T4euCp9/y7gBjMzd/ccZi0P7qFAd+3a+9bQ8Pnnmrvt3BlKetu28DXJ/Q8+yJ6nV69Q2CeeCOefH+43lnjfvho3FylRScq9P7Ah43E9cHxz67h7g5m9D/QC3spFyL3MnQvXXbfncdN/P7L9e5JtncbnWrrf3LLdu8Mt836S5/Lx713nztC9O3TrFm7du0OfPuGIlcbnM5f37AlDhoQC79kz93lEJLok5Z5t161pQyVZBzObCkwFOOSQQxL86Cx69w7XI9n7hff9uLl1Gp9r6X62ZVVV4X6HDnvfkj7XqdOeW8eOez9u6da5856i7tZNH2KKyOckKfd6YGDG4wHAxmbWqTezjsABwDtNX8jd5wBzAKqrq9u2CztuXLiJiEizklw4bBkw3MyGmFlnYDKwqMk6i4AL0vcnAI9pvF1EJJ4W99zTY+jTgMWEQyHnuvsaM5sFLHf3RcCvgXlmVkfYY5+cz9AiIrJviY5zd/daoLbJczMz7n8CnJvbaCIi0la6nruISBlSuYuIlCGVu4hIGVK5i4iUIZW7iEgZsliHo5vZVuD1Nn57b/JxaYPcUb72Ub72K/aMytd2g9y9T0srRSv39jCz5e5eHTtHc5SvfZSv/Yo9o/Lln4ZlRETKkMpdRKQMlWq5z4kdoAXK1z7K137FnlH58qwkx9xFRGTfSnXPXURE9qFoy93MzjWzNWa228yqmyy7zMzqzGytmX29me8fYmZPm9k6M7sjfbnifGW9w8yeS99eM7PnmlnvNTN7Pr3e8nzlyfJzrzKz/8nIOKaZ9WrS27TOzKYXMN9/mNlLZrbKzO41sx7NrFfQ7dfS9jCzLunffV36vTY435kyfvZAM3vczF5M/z35UZZ1TjWz9zN+7zOzvVYeM+7z92XBL9Pbb5WZHVPAbIdlbJfnzGybmf1Tk3Wibr92c/eivAF/DRwGLAGqM54fAawEugBDgFeAqizfvxCYnL5/M/CDAuW+HpjZzLLXgN4RtuVVwCUtrFOV3pZDgc7pbTyiQPnOBDqm718LXBt7+yXZHsDFwM3p+5OBOwr4O+0HHJO+3w14OUu+U4EHC/1+S/r7AsYADxFmcjsBeDpSzirgTcLx40Wz/dp7K9o9d3d/0d3XZlk0Hljg7jvc/VWgjjCJ92fMzICvESbrBrgd+GY+82b83InA/Hz/rDz4bCJ0d98JNE6Ennfu/gd3b0g/XEqY7Su2JNtjPOG9BeG9dnr6PZB37r7J3Z9N398OvEiYy7iUjAd+48FSoIeZ9YuQ43TgFXdv60mVRaloy30fsk3Y3fRN3Qt4L6Mwsq2TD18BNrv7umaWO/AHM1uRnk+2kKal/+s718yyzYqdZLsWwncJe3PZFHL7Jdkee00MDzRODF9Q6eGgo4GnsyweZWYrzewhMzuioMFa/n0Vy3tuMs3vkMXcfu2SaLKOfDGzR4C/yrJohrvf39y3ZXmuTRN2t0bCrFPY9177Se6+0cz6Ag+b2Uvu/kR7ciXJB9wEXE3YBlcTho6+2/Qlsnxvzg6lSrL9zGwG0AD8rpmXydv2yyLK+6y1zGx/4G7gn9x9W5PFzxKGGj5If85yHzC8gPFa+n0Vw/brDIwDLsuyOPb2a5eo5e7uZ7Th25JM2P0W4b94HdN7VNnWaZWWslqYGPwc4Nh9vMbG9NctZnYv4b/+OSmnpNvSzG4FHsyyKMl2bbME2+8CYCxwuqcHPLO8Rt62XxY5mxg+X8ysE6HYf+fu9zRdnln27l5rZjeaWW93L8g1UxL8vvL6nktoNPCsu29uuiD29muvUhyWWQRMTh+pMITwL+kzmSuky+FxwmTdECbvbu5/ArlyBvCSu9dnW2hmXzSzbo33CR8irs5zpsafnTmOeXYzPzfJROj5ylcDXAqMc/ePmlmn0NuvqCeGT4/t/xp40d3/bzPr/FXjZwBmNpLw9/3tAuVL8vtaBJyfPmrmBOB9d99UiHwZmv3fdsztlxOxP9Ft7kYooXpgB7AZWJyxbAbhSIa1wOiM52uBg9P3hxJKvw64E+iS57y3Ad9v8tzBQG1GnpXp2xrCcEShtuU84HlgFeEvVL+m+dKPxxCOunilwPnqCGOvz6VvNzfNF2P7ZdsewCzCP0IAXdPvrbr0e21oAbfZyYQhjFUZ220M8P3G9yEwLb2tVhI+qD6xgPmy/r6a5DNgdnr7Pk/GUXEFyvgFQlkfkPFcUWy/XNx0hqqISBkqxWEZERFpgcpdRKQMqdxFRMqQyl1EpAyp3EVEypDKXUSkDKncRUTKkMpdRKQM/X/BQ48hRjF6OwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def sigmoid(x):\n",
    "    return 1/(1+np.exp(-x))\n",
    "example_x = np.arange(-10,10, step=1)\n",
    "example_y = sigmoid(example_x)\n",
    "plt.plot(example_x, example_y, c='r')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Regularied cost function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost(theta, x ,y, lambda_):\n",
    "    theta = np.expand_dims(theta, axis = 1)\n",
    "    product = np.dot(x,theta)\n",
    "    hypo1 = sigmoid(product)\n",
    "    hypo2 = 1-hypo1\n",
    "    log1 = np.log(hypo1)\n",
    "    log2 = np.log(hypo2)\n",
    "    sum1 = y*log1\n",
    "    sum2 = (1-y)*log2\n",
    "    sum_values = -(sum1+sum2)\n",
    "    sum_theta = np.power(theta, 2)\n",
    "    m = len(x)\n",
    "    return np.sum(sum_values)/m+lambda_*np.sum(sum_theta)/(2*m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6931471805599454"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cost(theta, x, y, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient(theta, x, y ,lambda_):\n",
    "    theta = np.expand_dims(theta, axis = 1)\n",
    "    product = np.dot(x, theta)\n",
    "    hypo = sigmoid(product)\n",
    "    error = hypo-y\n",
    "    parameters = 28\n",
    "    gradient = []\n",
    "    for i in range(parameters):\n",
    "        if(i==0):\n",
    "            gradient.append(np.sum(error*x[:,i])/len(x))\n",
    "        else:\n",
    "            gradient.append(np.sum(error*x[:,i])/len(x)+lambda_*theta[i,0]/len(x))\n",
    "    grad = np.array(gradient)\n",
    "    return grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.00000000e+00,  5.47789085e-02,  1.83101559e-01,  2.47575335e-01,\n",
       "       -2.54718403e-02,  3.01369613e-01,  5.98333290e-02,  3.06815335e-02,\n",
       "        1.54825117e-02,  1.42350013e-01,  1.22538429e-01, -5.25103811e-03,\n",
       "        5.04328744e-02, -1.10481882e-02,  1.71098505e-01,  5.19650721e-02,\n",
       "        1.18117999e-02,  9.43209446e-03,  1.82778085e-02,  4.08908411e-03,\n",
       "        1.15709633e-01,  7.83711827e-02, -7.02782670e-04,  1.89334033e-02,\n",
       "       -1.70495724e-03,  2.25916984e-02, -6.30180777e-03,  1.25725602e-01])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradient(theta, x, y, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([-1.12624377e-15,  2.96392760e-17,  9.90632545e-17,  1.33948644e-16,\n",
       "        -1.37822641e-17,  1.63061692e-16,  3.23680766e-17,  1.65994571e-17,\n",
       "         8.37617996e-18,  7.70044691e-17,  6.63017498e-17, -2.84095237e-18,\n",
       "         2.72900778e-17, -5.97745404e-18,  9.25632578e-17,  2.81168324e-17,\n",
       "         6.39085213e-18,  5.10257342e-18,  9.88828764e-18,  2.21250189e-18,\n",
       "         6.26027755e-17,  4.24009020e-17, -3.80236315e-19,  1.02426899e-17,\n",
       "        -9.22404478e-19,  1.22220856e-17, -3.40960528e-18,  6.80166480e-17]),\n",
       " 50,\n",
       " 1)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import scipy.optimize as opt\n",
    "result = opt.fmin_tnc(func = cost, x0 = theta, fprime = gradient, args = (x,y,1))\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "T = result[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(theta, x):\n",
    "    theta = np.expand_dims(theta, axis = 1)\n",
    "    product = np.dot(x, theta)\n",
    "    hypo = sigmoid(product)\n",
    "    return (1 if i>=0.5 else 0 for i in hypo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.5084745762711864\n"
     ]
    }
   ],
   "source": [
    "predictions = predict(T, x)\n",
    "result = [1 if((a==1 and b==1)or(a==0 and b==0)) else 0 for (a,b) in zip(predictions, y)]\n",
    "result = np.array(result)\n",
    "correct = np.sum(result==1)\n",
    "print(\"Accuracy:\",correct/len(result))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}