g2-forestcovertype-modelparams-notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "32d729ac",
   "metadata": {
    "papermill": {
     "duration": 0.0347,
     "end_time": "2021-11-28T18:18:52.139498",
     "exception": false,
     "start_time": "2021-11-28T18:18:52.104798",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Import Statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c839aa74",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:18:52.222660Z",
     "iopub.status.busy": "2021-11-28T18:18:52.221729Z",
     "iopub.status.idle": "2021-11-28T18:18:55.505589Z",
     "shell.execute_reply": "2021-11-28T18:18:55.504670Z",
     "shell.execute_reply.started": "2021-11-28T04:03:10.510682Z"
    },
    "papermill": {
     "duration": 3.331976,
     "end_time": "2021-11-28T18:18:55.505754",
     "exception": false,
     "start_time": "2021-11-28T18:18:52.173778",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np, pandas as pd, matplotlib.pyplot as plt\n",
    "import joblib\n",
    "import optuna\n",
    "import sklearn \n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import confusion_matrix, classification_report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec025f02",
   "metadata": {
    "papermill": {
     "duration": 0.033314,
     "end_time": "2021-11-28T18:18:55.573358",
     "exception": false,
     "start_time": "2021-11-28T18:18:55.540044",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c5aa6aad",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:18:55.643767Z",
     "iopub.status.busy": "2021-11-28T18:18:55.643240Z",
     "iopub.status.idle": "2021-11-28T18:18:55.748556Z",
     "shell.execute_reply": "2021-11-28T18:18:55.749220Z",
     "shell.execute_reply.started": "2021-11-28T04:03:14.186078Z"
    },
    "papermill": {
     "duration": 0.142547,
     "end_time": "2021-11-28T18:18:55.749379",
     "exception": false,
     "start_time": "2021-11-28T18:18:55.606832",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Elevation</th>\n",
       "      <th>Aspect</th>\n",
       "      <th>Slope</th>\n",
       "      <th>Horizontal_Distance_To_Hydrology</th>\n",
       "      <th>Vertical_Distance_To_Hydrology</th>\n",
       "      <th>Horizontal_Distance_To_Roadways</th>\n",
       "      <th>Hillshade_9am</th>\n",
       "      <th>Hillshade_Noon</th>\n",
       "      <th>Hillshade_3pm</th>\n",
       "      <th>...</th>\n",
       "      <th>Soil_Type32</th>\n",
       "      <th>Soil_Type33</th>\n",
       "      <th>Soil_Type34</th>\n",
       "      <th>Soil_Type35</th>\n",
       "      <th>Soil_Type36</th>\n",
       "      <th>Soil_Type37</th>\n",
       "      <th>Soil_Type38</th>\n",
       "      <th>Soil_Type39</th>\n",
       "      <th>Soil_Type40</th>\n",
       "      <th>Cover_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2596</td>\n",
       "      <td>51</td>\n",
       "      <td>3</td>\n",
       "      <td>258</td>\n",
       "      <td>0</td>\n",
       "      <td>510</td>\n",
       "      <td>221</td>\n",
       "      <td>232</td>\n",
       "      <td>148</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2590</td>\n",
       "      <td>56</td>\n",
       "      <td>2</td>\n",
       "      <td>212</td>\n",
       "      <td>-6</td>\n",
       "      <td>390</td>\n",
       "      <td>220</td>\n",
       "      <td>235</td>\n",
       "      <td>151</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2804</td>\n",
       "      <td>139</td>\n",
       "      <td>9</td>\n",
       "      <td>268</td>\n",
       "      <td>65</td>\n",
       "      <td>3180</td>\n",
       "      <td>234</td>\n",
       "      <td>238</td>\n",
       "      <td>135</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2785</td>\n",
       "      <td>155</td>\n",
       "      <td>18</td>\n",
       "      <td>242</td>\n",
       "      <td>118</td>\n",
       "      <td>3090</td>\n",
       "      <td>238</td>\n",
       "      <td>238</td>\n",
       "      <td>122</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2595</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>153</td>\n",
       "      <td>-1</td>\n",
       "      <td>391</td>\n",
       "      <td>220</td>\n",
       "      <td>234</td>\n",
       "      <td>150</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 56 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  Elevation  Aspect  Slope  Horizontal_Distance_To_Hydrology  \\\n",
       "0   1       2596      51      3                               258   \n",
       "1   2       2590      56      2                               212   \n",
       "2   3       2804     139      9                               268   \n",
       "3   4       2785     155     18                               242   \n",
       "4   5       2595      45      2                               153   \n",
       "\n",
       "   Vertical_Distance_To_Hydrology  Horizontal_Distance_To_Roadways  \\\n",
       "0                               0                              510   \n",
       "1                              -6                              390   \n",
       "2                              65                             3180   \n",
       "3                             118                             3090   \n",
       "4                              -1                              391   \n",
       "\n",
       "   Hillshade_9am  Hillshade_Noon  Hillshade_3pm  ...  Soil_Type32  \\\n",
       "0            221             232            148  ...            0   \n",
       "1            220             235            151  ...            0   \n",
       "2            234             238            135  ...            0   \n",
       "3            238             238            122  ...            0   \n",
       "4            220             234            150  ...            0   \n",
       "\n",
       "   Soil_Type33  Soil_Type34  Soil_Type35  Soil_Type36  Soil_Type37  \\\n",
       "0            0            0            0            0            0   \n",
       "1            0            0            0            0            0   \n",
       "2            0            0            0            0            0   \n",
       "3            0            0            0            0            0   \n",
       "4            0            0            0            0            0   \n",
       "\n",
       "   Soil_Type38  Soil_Type39  Soil_Type40  Cover_Type  \n",
       "0            0            0            0           5  \n",
       "1            0            0            0           5  \n",
       "2            0            0            0           2  \n",
       "3            0            0            0           2  \n",
       "4            0            0            0           5  \n",
       "\n",
       "[5 rows x 56 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load data\n",
    "train = pd.read_csv('../input/forest-cover-type-prediction/train.csv')\n",
    "# view data\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f4a0d769",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:18:55.838374Z",
     "iopub.status.busy": "2021-11-28T18:18:55.836961Z",
     "iopub.status.idle": "2021-11-28T18:18:55.840987Z",
     "shell.execute_reply": "2021-11-28T18:18:55.840489Z",
     "shell.execute_reply.started": "2021-11-28T04:03:14.928006Z"
    },
    "papermill": {
     "duration": 0.056928,
     "end_time": "2021-11-28T18:18:55.841112",
     "exception": false,
     "start_time": "2021-11-28T18:18:55.784184",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Elevation</th>\n",
       "      <th>Aspect</th>\n",
       "      <th>Slope</th>\n",
       "      <th>Horizontal_Distance_To_Hydrology</th>\n",
       "      <th>Vertical_Distance_To_Hydrology</th>\n",
       "      <th>Horizontal_Distance_To_Roadways</th>\n",
       "      <th>Hillshade_9am</th>\n",
       "      <th>Hillshade_Noon</th>\n",
       "      <th>Hillshade_3pm</th>\n",
       "      <th>Horizontal_Distance_To_Fire_Points</th>\n",
       "      <th>...</th>\n",
       "      <th>Soil_Type32</th>\n",
       "      <th>Soil_Type33</th>\n",
       "      <th>Soil_Type34</th>\n",
       "      <th>Soil_Type35</th>\n",
       "      <th>Soil_Type36</th>\n",
       "      <th>Soil_Type37</th>\n",
       "      <th>Soil_Type38</th>\n",
       "      <th>Soil_Type39</th>\n",
       "      <th>Soil_Type40</th>\n",
       "      <th>Cover_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2596</td>\n",
       "      <td>51</td>\n",
       "      <td>3</td>\n",
       "      <td>258</td>\n",
       "      <td>0</td>\n",
       "      <td>510</td>\n",
       "      <td>221</td>\n",
       "      <td>232</td>\n",
       "      <td>148</td>\n",
       "      <td>6279</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2590</td>\n",
       "      <td>56</td>\n",
       "      <td>2</td>\n",
       "      <td>212</td>\n",
       "      <td>-6</td>\n",
       "      <td>390</td>\n",
       "      <td>220</td>\n",
       "      <td>235</td>\n",
       "      <td>151</td>\n",
       "      <td>6225</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2804</td>\n",
       "      <td>139</td>\n",
       "      <td>9</td>\n",
       "      <td>268</td>\n",
       "      <td>65</td>\n",
       "      <td>3180</td>\n",
       "      <td>234</td>\n",
       "      <td>238</td>\n",
       "      <td>135</td>\n",
       "      <td>6121</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2785</td>\n",
       "      <td>155</td>\n",
       "      <td>18</td>\n",
       "      <td>242</td>\n",
       "      <td>118</td>\n",
       "      <td>3090</td>\n",
       "      <td>238</td>\n",
       "      <td>238</td>\n",
       "      <td>122</td>\n",
       "      <td>6211</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2595</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>153</td>\n",
       "      <td>-1</td>\n",
       "      <td>391</td>\n",
       "      <td>220</td>\n",
       "      <td>234</td>\n",
       "      <td>150</td>\n",
       "      <td>6172</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 55 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Elevation  Aspect  Slope  Horizontal_Distance_To_Hydrology  \\\n",
       "0       2596      51      3                               258   \n",
       "1       2590      56      2                               212   \n",
       "2       2804     139      9                               268   \n",
       "3       2785     155     18                               242   \n",
       "4       2595      45      2                               153   \n",
       "\n",
       "   Vertical_Distance_To_Hydrology  Horizontal_Distance_To_Roadways  \\\n",
       "0                               0                              510   \n",
       "1                              -6                              390   \n",
       "2                              65                             3180   \n",
       "3                             118                             3090   \n",
       "4                              -1                              391   \n",
       "\n",
       "   Hillshade_9am  Hillshade_Noon  Hillshade_3pm  \\\n",
       "0            221             232            148   \n",
       "1            220             235            151   \n",
       "2            234             238            135   \n",
       "3            238             238            122   \n",
       "4            220             234            150   \n",
       "\n",
       "   Horizontal_Distance_To_Fire_Points  ...  Soil_Type32  Soil_Type33  \\\n",
       "0                                6279  ...            0            0   \n",
       "1                                6225  ...            0            0   \n",
       "2                                6121  ...            0            0   \n",
       "3                                6211  ...            0            0   \n",
       "4                                6172  ...            0            0   \n",
       "\n",
       "   Soil_Type34  Soil_Type35  Soil_Type36  Soil_Type37  Soil_Type38  \\\n",
       "0            0            0            0            0            0   \n",
       "1            0            0            0            0            0   \n",
       "2            0            0            0            0            0   \n",
       "3            0            0            0            0            0   \n",
       "4            0            0            0            0            0   \n",
       "\n",
       "   Soil_Type39  Soil_Type40  Cover_Type  \n",
       "0            0            0           5  \n",
       "1            0            0           5  \n",
       "2            0            0           2  \n",
       "3            0            0           2  \n",
       "4            0            0           5  \n",
       "\n",
       "[5 rows x 55 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove ID column from set\n",
    "train = train.iloc[:, 1:]\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0bd0275c",
   "metadata": {
    "papermill": {
     "duration": 0.039086,
     "end_time": "2021-11-28T18:18:55.918517",
     "exception": false,
     "start_time": "2021-11-28T18:18:55.879431",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Add Features from Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "093fbb4b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:18:56.046989Z",
     "iopub.status.busy": "2021-11-28T18:18:56.046033Z",
     "iopub.status.idle": "2021-11-28T18:18:56.055447Z",
     "shell.execute_reply": "2021-11-28T18:18:56.055821Z",
     "shell.execute_reply.started": "2021-11-28T04:03:16.318241Z"
    },
    "papermill": {
     "duration": 0.098989,
     "end_time": "2021-11-28T18:18:56.055977",
     "exception": false,
     "start_time": "2021-11-28T18:18:55.956988",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# add new features from feature engineering\n",
    "train['Elev_to_Horizontal_Hyd'] = train.Elevation - 0.2 * train.Horizontal_Distance_To_Hydrology \n",
    "train['Elev_to_Horizontal_Road'] = train.Elevation - 0.05 * train.Horizontal_Distance_To_Roadways  \n",
    "train['Elev_to_Verticle_Hyd'] = train.Elevation - train.Vertical_Distance_To_Hydrology \n",
    "train['Mean_Horizontal_Dist'] = (train.Horizontal_Distance_To_Fire_Points + train.Horizontal_Distance_To_Hydrology + \n",
    "                                 train.Horizontal_Distance_To_Roadways)/3 \n",
    "train['Mean_Fire_Hydro'] = (train.Horizontal_Distance_To_Fire_Points + train.Horizontal_Distance_To_Hydrology)/2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98cff140",
   "metadata": {
    "papermill": {
     "duration": 0.034209,
     "end_time": "2021-11-28T18:18:56.124804",
     "exception": false,
     "start_time": "2021-11-28T18:18:56.090595",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1ba828f1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:18:56.197512Z",
     "iopub.status.busy": "2021-11-28T18:18:56.196670Z",
     "iopub.status.idle": "2021-11-28T18:18:56.201466Z",
     "shell.execute_reply": "2021-11-28T18:18:56.201845Z",
     "shell.execute_reply.started": "2021-11-28T04:03:17.703045Z"
    },
    "papermill": {
     "duration": 0.042965,
     "end_time": "2021-11-28T18:18:56.201990",
     "exception": false,
     "start_time": "2021-11-28T18:18:56.159025",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check for missing values\n",
    "train.isnull().values.any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4103a455",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:18:56.279105Z",
     "iopub.status.busy": "2021-11-28T18:18:56.277983Z",
     "iopub.status.idle": "2021-11-28T18:18:56.423053Z",
     "shell.execute_reply": "2021-11-28T18:18:56.423547Z",
     "shell.execute_reply.started": "2021-11-28T04:03:18.403882Z"
    },
    "papermill": {
     "duration": 0.186858,
     "end_time": "2021-11-28T18:18:56.423701",
     "exception": false,
     "start_time": "2021-11-28T18:18:56.236843",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Elevation</th>\n",
       "      <th>Aspect</th>\n",
       "      <th>Slope</th>\n",
       "      <th>Horizontal_Distance_To_Hydrology</th>\n",
       "      <th>Vertical_Distance_To_Hydrology</th>\n",
       "      <th>Horizontal_Distance_To_Roadways</th>\n",
       "      <th>Hillshade_9am</th>\n",
       "      <th>Hillshade_Noon</th>\n",
       "      <th>Hillshade_3pm</th>\n",
       "      <th>Horizontal_Distance_To_Fire_Points</th>\n",
       "      <th>...</th>\n",
       "      <th>Soil_Type37</th>\n",
       "      <th>Soil_Type38</th>\n",
       "      <th>Soil_Type39</th>\n",
       "      <th>Soil_Type40</th>\n",
       "      <th>Cover_Type</th>\n",
       "      <th>Elev_to_Horizontal_Hyd</th>\n",
       "      <th>Elev_to_Horizontal_Road</th>\n",
       "      <th>Elev_to_Verticle_Hyd</th>\n",
       "      <th>Mean_Horizontal_Dist</th>\n",
       "      <th>Mean_Fire_Hydro</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "      <td>15120.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2749.322553</td>\n",
       "      <td>156.676653</td>\n",
       "      <td>16.501587</td>\n",
       "      <td>227.195701</td>\n",
       "      <td>51.076521</td>\n",
       "      <td>1714.023214</td>\n",
       "      <td>212.704299</td>\n",
       "      <td>218.965608</td>\n",
       "      <td>135.091997</td>\n",
       "      <td>1511.147288</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002249</td>\n",
       "      <td>0.048148</td>\n",
       "      <td>0.043452</td>\n",
       "      <td>0.030357</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2703.883413</td>\n",
       "      <td>2663.621392</td>\n",
       "      <td>2698.246032</td>\n",
       "      <td>1150.788735</td>\n",
       "      <td>869.171495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>417.678187</td>\n",
       "      <td>110.085801</td>\n",
       "      <td>8.453927</td>\n",
       "      <td>210.075296</td>\n",
       "      <td>61.239406</td>\n",
       "      <td>1325.066358</td>\n",
       "      <td>30.561287</td>\n",
       "      <td>22.801966</td>\n",
       "      <td>45.895189</td>\n",
       "      <td>1099.936493</td>\n",
       "      <td>...</td>\n",
       "      <td>0.047368</td>\n",
       "      <td>0.214086</td>\n",
       "      <td>0.203880</td>\n",
       "      <td>0.171574</td>\n",
       "      <td>2.000066</td>\n",
       "      <td>402.163115</td>\n",
       "      <td>383.169169</td>\n",
       "      <td>414.680029</td>\n",
       "      <td>716.025159</td>\n",
       "      <td>576.061501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1863.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-146.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1839.000000</td>\n",
       "      <td>1858.500000</td>\n",
       "      <td>1842.000000</td>\n",
       "      <td>38.333333</td>\n",
       "      <td>15.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2376.000000</td>\n",
       "      <td>65.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>67.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>764.000000</td>\n",
       "      <td>196.000000</td>\n",
       "      <td>207.000000</td>\n",
       "      <td>106.000000</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2338.750000</td>\n",
       "      <td>2336.250000</td>\n",
       "      <td>2323.000000</td>\n",
       "      <td>652.583333</td>\n",
       "      <td>451.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2752.000000</td>\n",
       "      <td>126.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>180.000000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>1316.000000</td>\n",
       "      <td>220.000000</td>\n",
       "      <td>223.000000</td>\n",
       "      <td>138.000000</td>\n",
       "      <td>1256.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2711.500000</td>\n",
       "      <td>2676.600000</td>\n",
       "      <td>2696.000000</td>\n",
       "      <td>920.166667</td>\n",
       "      <td>731.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>3104.000000</td>\n",
       "      <td>261.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>330.000000</td>\n",
       "      <td>79.000000</td>\n",
       "      <td>2270.000000</td>\n",
       "      <td>235.000000</td>\n",
       "      <td>235.000000</td>\n",
       "      <td>167.000000</td>\n",
       "      <td>1988.250000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>3049.250000</td>\n",
       "      <td>2974.750000</td>\n",
       "      <td>3068.000000</td>\n",
       "      <td>1524.750000</td>\n",
       "      <td>1151.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3849.000000</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>1343.000000</td>\n",
       "      <td>554.000000</td>\n",
       "      <td>6890.000000</td>\n",
       "      <td>254.000000</td>\n",
       "      <td>254.000000</td>\n",
       "      <td>248.000000</td>\n",
       "      <td>6993.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>3670.800000</td>\n",
       "      <td>3626.800000</td>\n",
       "      <td>3776.000000</td>\n",
       "      <td>4361.000000</td>\n",
       "      <td>3583.500000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Elevation        Aspect         Slope  \\\n",
       "count  15120.000000  15120.000000  15120.000000   \n",
       "mean    2749.322553    156.676653     16.501587   \n",
       "std      417.678187    110.085801      8.453927   \n",
       "min     1863.000000      0.000000      0.000000   \n",
       "25%     2376.000000     65.000000     10.000000   \n",
       "50%     2752.000000    126.000000     15.000000   \n",
       "75%     3104.000000    261.000000     22.000000   \n",
       "max     3849.000000    360.000000     52.000000   \n",
       "\n",
       "       Horizontal_Distance_To_Hydrology  Vertical_Distance_To_Hydrology  \\\n",
       "count                      15120.000000                    15120.000000   \n",
       "mean                         227.195701                       51.076521   \n",
       "std                          210.075296                       61.239406   \n",
       "min                            0.000000                     -146.000000   \n",
       "25%                           67.000000                        5.000000   \n",
       "50%                          180.000000                       32.000000   \n",
       "75%                          330.000000                       79.000000   \n",
       "max                         1343.000000                      554.000000   \n",
       "\n",
       "       Horizontal_Distance_To_Roadways  Hillshade_9am  Hillshade_Noon  \\\n",
       "count                     15120.000000   15120.000000    15120.000000   \n",
       "mean                       1714.023214     212.704299      218.965608   \n",
       "std                        1325.066358      30.561287       22.801966   \n",
       "min                           0.000000       0.000000       99.000000   \n",
       "25%                         764.000000     196.000000      207.000000   \n",
       "50%                        1316.000000     220.000000      223.000000   \n",
       "75%                        2270.000000     235.000000      235.000000   \n",
       "max                        6890.000000     254.000000      254.000000   \n",
       "\n",
       "       Hillshade_3pm  Horizontal_Distance_To_Fire_Points  ...   Soil_Type37  \\\n",
       "count   15120.000000                        15120.000000  ...  15120.000000   \n",
       "mean      135.091997                         1511.147288  ...      0.002249   \n",
       "std        45.895189                         1099.936493  ...      0.047368   \n",
       "min         0.000000                            0.000000  ...      0.000000   \n",
       "25%       106.000000                          730.000000  ...      0.000000   \n",
       "50%       138.000000                         1256.000000  ...      0.000000   \n",
       "75%       167.000000                         1988.250000  ...      0.000000   \n",
       "max       248.000000                         6993.000000  ...      1.000000   \n",
       "\n",
       "        Soil_Type38   Soil_Type39   Soil_Type40    Cover_Type  \\\n",
       "count  15120.000000  15120.000000  15120.000000  15120.000000   \n",
       "mean       0.048148      0.043452      0.030357      4.000000   \n",
       "std        0.214086      0.203880      0.171574      2.000066   \n",
       "min        0.000000      0.000000      0.000000      1.000000   \n",
       "25%        0.000000      0.000000      0.000000      2.000000   \n",
       "50%        0.000000      0.000000      0.000000      4.000000   \n",
       "75%        0.000000      0.000000      0.000000      6.000000   \n",
       "max        1.000000      1.000000      1.000000      7.000000   \n",
       "\n",
       "       Elev_to_Horizontal_Hyd  Elev_to_Horizontal_Road  Elev_to_Verticle_Hyd  \\\n",
       "count            15120.000000             15120.000000          15120.000000   \n",
       "mean              2703.883413              2663.621392           2698.246032   \n",
       "std                402.163115               383.169169            414.680029   \n",
       "min               1839.000000              1858.500000           1842.000000   \n",
       "25%               2338.750000              2336.250000           2323.000000   \n",
       "50%               2711.500000              2676.600000           2696.000000   \n",
       "75%               3049.250000              2974.750000           3068.000000   \n",
       "max               3670.800000              3626.800000           3776.000000   \n",
       "\n",
       "       Mean_Horizontal_Dist  Mean_Fire_Hydro  \n",
       "count          15120.000000     15120.000000  \n",
       "mean            1150.788735       869.171495  \n",
       "std              716.025159       576.061501  \n",
       "min               38.333333        15.000000  \n",
       "25%              652.583333       451.500000  \n",
       "50%              920.166667       731.000000  \n",
       "75%             1524.750000      1151.000000  \n",
       "max             4361.000000      3583.500000  \n",
       "\n",
       "[8 rows x 60 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# summary\n",
    "train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "248b4024",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:18:56.503109Z",
     "iopub.status.busy": "2021-11-28T18:18:56.502483Z",
     "iopub.status.idle": "2021-11-28T18:18:56.505550Z",
     "shell.execute_reply": "2021-11-28T18:18:56.506052Z",
     "shell.execute_reply.started": "2021-11-28T04:03:19.099721Z"
    },
    "papermill": {
     "duration": 0.044239,
     "end_time": "2021-11-28T18:18:56.506177",
     "exception": false,
     "start_time": "2021-11-28T18:18:56.461938",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15120, 60)\n",
      "Index(['Elevation', 'Aspect', 'Slope', 'Horizontal_Distance_To_Hydrology',\n",
      "       'Vertical_Distance_To_Hydrology', 'Horizontal_Distance_To_Roadways',\n",
      "       'Hillshade_9am', 'Hillshade_Noon', 'Hillshade_3pm',\n",
      "       'Horizontal_Distance_To_Fire_Points', 'Wilderness_Area1',\n",
      "       'Wilderness_Area2', 'Wilderness_Area3', 'Wilderness_Area4',\n",
      "       'Soil_Type1', 'Soil_Type2', 'Soil_Type3', 'Soil_Type4', 'Soil_Type5',\n",
      "       'Soil_Type6', 'Soil_Type7', 'Soil_Type8', 'Soil_Type9', 'Soil_Type10',\n",
      "       'Soil_Type11', 'Soil_Type12', 'Soil_Type13', 'Soil_Type14',\n",
      "       'Soil_Type15', 'Soil_Type16', 'Soil_Type17', 'Soil_Type18',\n",
      "       'Soil_Type19', 'Soil_Type20', 'Soil_Type21', 'Soil_Type22',\n",
      "       'Soil_Type23', 'Soil_Type24', 'Soil_Type25', 'Soil_Type26',\n",
      "       'Soil_Type27', 'Soil_Type28', 'Soil_Type29', 'Soil_Type30',\n",
      "       'Soil_Type31', 'Soil_Type32', 'Soil_Type33', 'Soil_Type34',\n",
      "       'Soil_Type35', 'Soil_Type36', 'Soil_Type37', 'Soil_Type38',\n",
      "       'Soil_Type39', 'Soil_Type40', 'Cover_Type', 'Elev_to_Horizontal_Hyd',\n",
      "       'Elev_to_Horizontal_Road', 'Elev_to_Verticle_Hyd',\n",
      "       'Mean_Horizontal_Dist', 'Mean_Fire_Hydro'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "# dimensions of data set \n",
    "print(train.shape) # 60 columns\n",
    "# column names\n",
    "print(train.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12cb2084",
   "metadata": {
    "papermill": {
     "duration": 0.035515,
     "end_time": "2021-11-28T18:18:56.578275",
     "exception": false,
     "start_time": "2021-11-28T18:18:56.542760",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "95546e3d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:18:56.668788Z",
     "iopub.status.busy": "2021-11-28T18:18:56.660597Z",
     "iopub.status.idle": "2021-11-28T18:18:56.671498Z",
     "shell.execute_reply": "2021-11-28T18:18:56.671924Z",
     "shell.execute_reply.started": "2021-11-28T04:03:20.611779Z"
    },
    "papermill": {
     "duration": 0.05758,
     "end_time": "2021-11-28T18:18:56.672049",
     "exception": false,
     "start_time": "2021-11-28T18:18:56.614469",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cover_Type</th>\n",
       "      <th>Elevation</th>\n",
       "      <th>Aspect</th>\n",
       "      <th>Slope</th>\n",
       "      <th>Horizontal_Distance_To_Hydrology</th>\n",
       "      <th>Vertical_Distance_To_Hydrology</th>\n",
       "      <th>Horizontal_Distance_To_Roadways</th>\n",
       "      <th>Hillshade_9am</th>\n",
       "      <th>Hillshade_Noon</th>\n",
       "      <th>Hillshade_3pm</th>\n",
       "      <th>...</th>\n",
       "      <th>Soil_Type36</th>\n",
       "      <th>Soil_Type37</th>\n",
       "      <th>Soil_Type38</th>\n",
       "      <th>Soil_Type39</th>\n",
       "      <th>Soil_Type40</th>\n",
       "      <th>Elev_to_Horizontal_Hyd</th>\n",
       "      <th>Elev_to_Horizontal_Road</th>\n",
       "      <th>Elev_to_Verticle_Hyd</th>\n",
       "      <th>Mean_Horizontal_Dist</th>\n",
       "      <th>Mean_Fire_Hydro</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>2596</td>\n",
       "      <td>51</td>\n",
       "      <td>3</td>\n",
       "      <td>258</td>\n",
       "      <td>0</td>\n",
       "      <td>510</td>\n",
       "      <td>221</td>\n",
       "      <td>232</td>\n",
       "      <td>148</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2544.4</td>\n",
       "      <td>2570.50</td>\n",
       "      <td>2596</td>\n",
       "      <td>2349.000000</td>\n",
       "      <td>3268.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>2590</td>\n",
       "      <td>56</td>\n",
       "      <td>2</td>\n",
       "      <td>212</td>\n",
       "      <td>-6</td>\n",
       "      <td>390</td>\n",
       "      <td>220</td>\n",
       "      <td>235</td>\n",
       "      <td>151</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2547.6</td>\n",
       "      <td>2570.50</td>\n",
       "      <td>2596</td>\n",
       "      <td>2275.666667</td>\n",
       "      <td>3218.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2804</td>\n",
       "      <td>139</td>\n",
       "      <td>9</td>\n",
       "      <td>268</td>\n",
       "      <td>65</td>\n",
       "      <td>3180</td>\n",
       "      <td>234</td>\n",
       "      <td>238</td>\n",
       "      <td>135</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2750.4</td>\n",
       "      <td>2645.00</td>\n",
       "      <td>2739</td>\n",
       "      <td>3189.666667</td>\n",
       "      <td>3194.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>2785</td>\n",
       "      <td>155</td>\n",
       "      <td>18</td>\n",
       "      <td>242</td>\n",
       "      <td>118</td>\n",
       "      <td>3090</td>\n",
       "      <td>238</td>\n",
       "      <td>238</td>\n",
       "      <td>122</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2736.6</td>\n",
       "      <td>2630.50</td>\n",
       "      <td>2667</td>\n",
       "      <td>3181.000000</td>\n",
       "      <td>3226.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2595</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>153</td>\n",
       "      <td>-1</td>\n",
       "      <td>391</td>\n",
       "      <td>220</td>\n",
       "      <td>234</td>\n",
       "      <td>150</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2564.4</td>\n",
       "      <td>2575.45</td>\n",
       "      <td>2596</td>\n",
       "      <td>2238.666667</td>\n",
       "      <td>3162.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Cover_Type  Elevation  Aspect  Slope  Horizontal_Distance_To_Hydrology  \\\n",
       "0           5       2596      51      3                               258   \n",
       "1           5       2590      56      2                               212   \n",
       "2           2       2804     139      9                               268   \n",
       "3           2       2785     155     18                               242   \n",
       "4           5       2595      45      2                               153   \n",
       "\n",
       "   Vertical_Distance_To_Hydrology  Horizontal_Distance_To_Roadways  \\\n",
       "0                               0                              510   \n",
       "1                              -6                              390   \n",
       "2                              65                             3180   \n",
       "3                             118                             3090   \n",
       "4                              -1                              391   \n",
       "\n",
       "   Hillshade_9am  Hillshade_Noon  Hillshade_3pm  ...  Soil_Type36  \\\n",
       "0            221             232            148  ...            0   \n",
       "1            220             235            151  ...            0   \n",
       "2            234             238            135  ...            0   \n",
       "3            238             238            122  ...            0   \n",
       "4            220             234            150  ...            0   \n",
       "\n",
       "   Soil_Type37  Soil_Type38  Soil_Type39  Soil_Type40  Elev_to_Horizontal_Hyd  \\\n",
       "0            0            0            0            0                  2544.4   \n",
       "1            0            0            0            0                  2547.6   \n",
       "2            0            0            0            0                  2750.4   \n",
       "3            0            0            0            0                  2736.6   \n",
       "4            0            0            0            0                  2564.4   \n",
       "\n",
       "   Elev_to_Horizontal_Road  Elev_to_Verticle_Hyd  Mean_Horizontal_Dist  \\\n",
       "0                  2570.50                  2596           2349.000000   \n",
       "1                  2570.50                  2596           2275.666667   \n",
       "2                  2645.00                  2739           3189.666667   \n",
       "3                  2630.50                  2667           3181.000000   \n",
       "4                  2575.45                  2596           2238.666667   \n",
       "\n",
       "   Mean_Fire_Hydro  \n",
       "0           3268.5  \n",
       "1           3218.5  \n",
       "2           3194.5  \n",
       "3           3226.5  \n",
       "4           3162.5  \n",
       "\n",
       "[5 rows x 60 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# move target to first column\n",
    "first_column = train.pop('Cover_Type')\n",
    "  \n",
    "# insert column using insert(position,column_name,first_column) function\n",
    "train.insert(0, 'Cover_Type', first_column)\n",
    "  \n",
    "# view\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6c61cec3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:18:56.756791Z",
     "iopub.status.busy": "2021-11-28T18:18:56.755865Z",
     "iopub.status.idle": "2021-11-28T18:18:56.760456Z",
     "shell.execute_reply": "2021-11-28T18:18:56.759952Z",
     "shell.execute_reply.started": "2021-11-28T04:03:21.681822Z"
    },
    "papermill": {
     "duration": 0.050516,
     "end_time": "2021-11-28T18:18:56.760575",
     "exception": false,
     "start_time": "2021-11-28T18:18:56.710059",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# create cat, num, and y\n",
    "X_cat = train.iloc[:, 11:55].values\n",
    "B = train.iloc[:, 55:60]\n",
    "A = train.iloc[:, 1:11]\n",
    "X_num = pd.concat([A, B], axis = 1).values\n",
    "y = train.iloc[:, 0].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8b137938",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:18:56.841067Z",
     "iopub.status.busy": "2021-11-28T18:18:56.839913Z",
     "iopub.status.idle": "2021-11-28T18:18:56.848894Z",
     "shell.execute_reply": "2021-11-28T18:18:56.848201Z",
     "shell.execute_reply.started": "2021-11-28T04:03:24.374033Z"
    },
    "papermill": {
     "duration": 0.051275,
     "end_time": "2021-11-28T18:18:56.849060",
     "exception": false,
     "start_time": "2021-11-28T18:18:56.797785",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Categorical Shape: (15120, 44)\n",
      "Numerical Shape: (15120, 15)\n",
      "Label Shape: (15120,)\n"
     ]
    }
   ],
   "source": [
    "# scale/standardizing numerical columns\n",
    "# scaler object\n",
    "scaler = StandardScaler()\n",
    "# fit to training data\n",
    "scaler.fit(X_num)\n",
    "# scale num columns\n",
    "X_num = scaler.transform(X_num)\n",
    "\n",
    "# shape\n",
    "print(f'Categorical Shape: {X_cat.shape}')\n",
    "print(f'Numerical Shape: {X_num.shape}')\n",
    "print(f'Label Shape: {y.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d1e04038",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:18:56.933013Z",
     "iopub.status.busy": "2021-11-28T18:18:56.931905Z",
     "iopub.status.idle": "2021-11-28T18:18:56.938553Z",
     "shell.execute_reply": "2021-11-28T18:18:56.939157Z",
     "shell.execute_reply.started": "2021-11-28T04:03:25.360078Z"
    },
    "papermill": {
     "duration": 0.049016,
     "end_time": "2021-11-28T18:18:56.939330",
     "exception": false,
     "start_time": "2021-11-28T18:18:56.890314",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15120, 59)\n"
     ]
    }
   ],
   "source": [
    "# combine num and cat\n",
    "X = np.hstack((X_num, X_cat))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b38f18b",
   "metadata": {
    "papermill": {
     "duration": 0.036536,
     "end_time": "2021-11-28T18:18:57.013664",
     "exception": false,
     "start_time": "2021-11-28T18:18:56.977128",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "46b3557d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:18:57.092156Z",
     "iopub.status.busy": "2021-11-28T18:18:57.091115Z",
     "iopub.status.idle": "2021-11-28T18:18:57.429396Z",
     "shell.execute_reply": "2021-11-28T18:18:57.428637Z",
     "shell.execute_reply.started": "2021-11-28T04:03:28.488011Z"
    },
    "papermill": {
     "duration": 0.379364,
     "end_time": "2021-11-28T18:18:57.429524",
     "exception": false,
     "start_time": "2021-11-28T18:18:57.050160",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'PCA Number of Components for Cumulative Variance')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEWCAYAAACEz/viAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAzUUlEQVR4nO3deXxcdb3/8dc7W9Om6b7RvYWylFUsiMoPAZVFL+BVvJQrsqjgvYrgeoUrInqv1+sOenEBRFlERK9LRQSRTa8otGAFytbShS4kTdc0SZM0yef3x/c77XRIJidpJpOZ+Twfj3nM2efzPXPmfOZs36/MDOecc6WtLN8BOOecyz9PBs455zwZOOec82TgnHMOTwbOOefwZOCccw5PBgVHkkk6IE+ffZCkpZJ2SLosHzEUs2Jev5Jmx223op/z/7ukmwY6roEmaZmkE/MdR38UXTKQtFrSTklNkuol/UjSyLTxp0r6Y/zBNUh6RNKZGcs4MW64n+7ls1LTfSdj+P9JunBACzY0/BvwkJnVmtm3upsgyfotZnH7e0s/Z+91/fYjngMl/UzSJknbJT0l6eOSygdi+bkQf1fr0oeZ2X+Z2QcG+HO+J+nWboYfKalN0ri+LtPMDjWzhwckwEFWdMkgOsPMRgJHAwuAqwAknQ38DLgVmA5MBq4GzsiY/wJgC3B+gs9qBt4rafaARD5I+vkPbRawLMsyk65f172s6zeb7r5PSfsDjwFrgcPNbDTwbsJvonYf4iwWtwDvlFSTMfy9wN1mtiXpgvp7xDOkmFlRvYDVwFvS+r8K3A0IeBn4VC/z1wA7gIVAO7Agy7QnAuuAbwM/TBv+f8CFsfsa4Pa0cbMBAypi/8PAfwKPAk3Ab4DxwI+BRmAxMDttfgMuA1YCm2L5ytLGvw94DtgK3AfMypj3w8ByYFUPZTqTsEPaFmM7JA5/EOgEWmOcB2bM1+v6Jfz5uApYA2wkJI3RGevlIsLOayvwL8AxwFMxnv9JW9aFwJ+B/wG2A88Db04bPxVYREjqK4CL08ZdA9wVP39HLO+CjHn/F2gAVgGXJZkXuA3oAnbGdfRvQDVwO7A5lmExMLmbdfOq9QuMjp/TENfZVanvOq3834zL/s9ulnk78Nvett+efj+xrD+Ly9kBPB3jujJ+f2uBU7L89q4hbvu8eru/iLCd7iBsyx9M+/3tjOuxKb6mZizrd8ClGXH/HXhn7D4YuD9+9y8A/5RlHbwAnJ/WXw5sAM4C9o/fy2bCb+3HwJiM8n6asH22ARUZ6+9Y4C/xe3+FsK1WZfwe/4Xwe9wGXA8obfzFaevoWeDo3rbPfdp3DsRChtIr48uYQfix/kfcQAyY08v8741fXDlhx/zt3n5MwBTCjvugOLyvyWBF3PBGxy/9ReAtceO6lb0TjQEPAeOAmXHaD8RxZ8VlHRLnvQp4NGPe++O8w7spz4GEI523ApWEndmK1AYcY/1AD+ui1/VLSFQrgLnASOAXwG0Z6+V7hB3oKYQd46+AScA0wg7oTXH6C4EO4GMx1nMISWFcHP9H4DtxWUfFH87Jad9JK/C2+D1/CfhrHFcGPEE4oqmKsa4ETu1t3sztL/Z/kLAdjYjTvxYY1cP62Wv9xu/+14R/8bPjd/3+jPJ/JH7X3X2fdcBFvW2/WX4/qbKeyp5tcRXwmbjOLybtT0U3Zb+GnpPB2wnbvIA3AS3s2dl1F1f6ss4H/pw2bj5hZzqMkEzWEpJNBfAawo58fg/r4DPAH9L6T43bSiVwAOG3MAyYSNimrs0o71LCfmZ4N+vvtcBxMY7ZhB37RzN+j3cDYwi/5QbgtDju3cB6wp8hxVhm0cv2uU/7zn1dwFB7xS+jKW4cawg7hOHAG+PKr+5l/j+kvnDg3NSG0duPCfgK8NPY3ddk8Jm08V8HfpfWfwawNGMDOi2t/0PAA7H7d8SdRewvI/zIZqXNe3KWsn8WuCtj/vXAiWmx9pQMel2/wAPAh9L6DwJ2pf1YDJiWNn4zcE5a//+mfkyEneEG9v4n9Tghmc8g/MuuTRv3JeBHad9J+g5gPrAzdr8OeDkj7iuJCTnbvGnbX/oO8X2Eo74jEmy7u9cvIXG0k7YTIySWh9PK/3Ivy9uVvq1k2367iz+W9f6MbbEJKI/9tfE7G9ND2a+hh2TQTSy/Ai7PElf6smoJf1pS2/UXgZtj9znAnzLm/T7wuR4+d2ZcT9Nj/4+B63qY9h3A3zLW1ft6Wn/dzP9R4Jdp/QYcn9Z/F3BF7L4vtT4ylpF1+9yXV7FeM3iHmY0xs1lm9iEz20nYsQDs19NMkmYAJxE2CAj/yqoJ/2J682XgVElH9iPe+rTund30j9x7ctamda8hHDZC+OdwnaRtkrYRDpNF+Ffd3byZpsblAWBmXXH6aT3OsUev6zdz+bG7gnBtIaUv62K9xV9D2vKmxtcWM9uRMS69HHVp3S1AdTzvOwuYmlqHcT3+e0aMPc3bndsIP+w7JW2Q9BVJlT1Mm24C4d9p5vpK+l1C+E6yfR9JZK7/TWbWmdYPr94+eyXpdEl/lbQlruO3Ecrcq/i9/pZwKhfCn7bUb3YW8LqM7+89hKP37pb1MuEf/3nxRpN3EI6AkDRZ0p2S1ktqJJwuy4yxx+8gXry/W1JdnP+/upk/c1tKrcsZwEvdLDbJ9tkvxZoMuvMC4Yt7V5Zp3ktYJ7+RVEc4/KomXFDOysw2A9cSTkmlayacIkjpdqPsoxlp3TMJ/5AhlO+DMRGmXsPN7NH0ULMsdwNhYwNAkuJnrU8QU5L1u9fyY+wd7L3D6YtpMcb05W2Ir3GSajPGJSnHWsKpj/R1WGtmb0sY017r18x2mdnnzWw+8AbgH0h2Y8Imwj/WzPWVXoZs3yWEo9xs38de22a8w2higtgSLY8etnVJwwhHeV8jXD8ZA9xD+OMCvZcL4CfAuZJeT/iNPhSHrwUeyfj+RprZv2ZZ1i2E3/67CN/9E3H4f8VYDjezUcB5aTGmZIv1u4RrWfPi/P/ezfw9WUs4jdbd8H3ZPntUMskg/oP8OPBZSRdJGiWpTNLxkm6Ik10AfJ5wjjn1ehfwNknjE3zMNwg/+EPShi0FTpA0U9JowiHdvvqUpLHxSOZy4Kdx+PeAKyUdCiBptKR392G5dwFvl/Tm+O/1E4QLY49mny3x+v0J8DFJc+K/sP8inFrr6EOM6SYBl0mqjOU8BLjHzNbGmL8kqVrSEcD7Cf/sevM4sEPSpyUNl1Qu6TBJxySMqZ5wHhcASSdJOjzuaBsJO/iu3hYS/33fBXxRUq2kWYT1m6QMKZ8D3iDpq5KmxHgOkHS7pDGEaxDVkt4ev++rCOfH+2spsDB+HwuAs3uYrip+TgPQIel0wjWilHpgfPy99OQeQqL8AmEbSq3Tu4EDJb03xlEp6RhJh/S4pJCYZhJ++7ekDa8lnBbbLmka8Kksy+hOLeE7b5J0MJAtIWW6CfikpNcqOCBuA/u6ffaoZJIBgJn9nHBO8X2Ef4/1hDt5fi3pOMLGdb2Z1aW9FhEuep6bYPmNhGsH49KG3U/YWT9FuPBz9wAU5ddxWUsJh8s/iJ/1S8LpqjvjYekzwOlJF2pmLxD+/Xyb8M/0DMJtuu0J5+9x/cZJbiacNvkj4UJkK+ECaH89BsyLsX4RODseoUH4vmbHOH5JOGf8hwRl6CT8ez8qxriJ8MPMtmNK9yXgqngI/0nCv+OfE3YKzwGPENZBEh8h/NteSbgOdQdhHSZiZi8Bryesh2WSthN2fEuAHWa2nXDN6SbCEUcz4YaI/vos4d/sVsKO9Y4e4tpBuCPurjjtPxPu/EqNf57wx2FlXI9Tu1lGG+EGhLekf05c9imEU0gbCKdhvkyWJGdmzYT1Mp09p5uIZTiacGPCb+Pn9cUnY9l2ADey509br8zsZ4Rt+o44/68IN0fs6/bZI+19ytW5wqDwUN8HzOz4fMfiXDEoqSMD55xz3fNk4Jxzzk8TOeec8yMD55xzhAd+CsqECRNs9uzZ+Q7DOecKyhNPPLHJzHp8jqTgksHs2bNZsmRJvsNwzrmCImlNtvF+msg555wnA+ecc54MnHPO4cnAOeccngycc86Rw2Qg6WZJGyU908N4SfqWpBUKjXQfnatYnHPOZZfLI4MfAadlGX86ocbJecAlhLq/nXPO5UHOnjMwsz9Kmp1lkrOAW2M9+H+VNEbSfmb2Sq5icq5YdXUZu7q66Og0OrqMzi6jo7Nrd3dnVxjeZUZHZ3yPw7ssvncZnZY+DLosDO8y6IzNI4ZhYZzF4aluM8MAsz3jQ3+o9ib0p3fvGZZeM47Znv7UNKF7z7ypcT0qwqp23nzIZI6cMSYny87nQ2fT2LvJuHVx2KuSgaRLCEcPzJw5c1CCc25fdHR20dzeSUt7B81tnexs72TnrtCf6t65Kwxv3dVJSxzW1tFF665O2nbF944u2jo6ae/ooq2ji/bOLto74iutu6Or+HZ8A0FJ2xUrEJNGVRdlMkjMzG4AbgBYsGCBb/Uu58yMlvZOtjS3s61lF9t2pt530bhzF9ta2mnc2cH2nbtobI2vnR00t3XQ1NZBW0evjZntpbJcVFeUM6yynOrKMoZVlFFdWc6wijKqKsoYM6KKqtg9rLxsT3d8ryovp7JCVJSJ8rKy+K497+WiTKKirIzyMkJ3HFZeJsolyuK0ew9jd3+ZRJn29Ct2lyl0SyD2DBd7hrF7PCiOg73nSe/f050avvc8qWFu4OQzGaxn77Z8p5OsjVrn+qWlvYPNTe00NLWxuamdzU1tbG5uZ1Oqv7mNLc272NrczpaWdtqz7NCrK8sYPbySUdWVjB5eyaTaavafWMHIYRXUDKugpqqCmmHl1AyrYERVOSOqKhheWc7wqnJGVJUzvDK8V8fuynK/sc/lVz6TwSLgUkl3Aq8Dtvv1AtdXHZ1dbGpqp76xlfrGVhqa2mjYEV6bmlLvYYff0t7Z7TJqh1UwbmQV42qqmDZmOIdPG8XYmirGjahibE0VY4ZX7n4fPSLs/IdVlA9ySZ3LrZwlA0k/AU4EJkhaR2icuxLAzL5HaND6bYT2hVuAi3IViytM7R1dvLJ9J+u27uSV7WFn/8r2ndRtb6WusZX6xrDD7+464biaKiaOHMaE2ipeM3MME0YOi68qJowcxvj4Pq6miupK37E7l8u7ibI2IB/vIvpwrj7fDX1mxubmdtZsbuHlLc2s3tTC2i0trNu6k7VbW6hrbH3Vjn7MiEqmjKpmyuhqDps6mkmjqpk8ahiTa6uZPKqaibVhR++nXZzrm4K4gOwK27aWdlZuamb1pmZWpb3WbG6hqa1j93QSTB09nOljh/OG/Scwfezw+BrBfqNDAvB/8c7lhicDNyA6Ort4eUsLLzU081JDEysbmljZ0MzKTc1saW7fPV2ZYMa4EcyZUMMxs8cxa/yI+Kph+tjhfi7euTzxZOD6pKW9g5c2NrN84w5WbGzipYYmXmpoZs3mZnZ17jmnM2HkMOZOrOHUQyczd8JI5kyoYc7EGmaMHUFVhZ/CcW6o8WTgutXS3sGKjU28ULeDF+t3sHxjE8vrm1i/befuaSrKxOwJNew/sYZT5k9m/4kjmTuxhrkTRzJ6eGUeo3fO9ZUngxLX0dnF6s0tvFC3g+frGnnulbDzX7u1ZffF26qKMvafOJIFs8eycOIM5k0eyQGTapk1foRfqHWuSHgyKCE72zt5rq6RZeu3s2xDI8s2NPJi/Y7dT8uWl4k5E2o4fPpozn7tdA6cPJIDJ9cya3wN5WX+tKdzxcyTQZEyM15qaOaJNVtYsnorS9du46WGJlJV2IwdUcmhU0dz/utncfCUURw0pZYDJo30u3WcK1GeDIrIuq0t3P9sPX9esZknX966+y6eMSMqOXrmWE4/fD8OmzqKw6aNZr/R1V63i3NuN08GBczMeLG+ifuW1fH7Z+t4Zn0jALPHj+DkgyexYNZYFswey9wJIynz0zzOuSw8GRSgzi7j7qc2cP1DK3ixvgmAo2eO4YrTD+bUQ6cwZ0JNniN0zhUaTwYFJJUErntgOSsbmjl4Si3/8Y7DOGX+ZCaPqs53eM65AubJoACYGb956hWu+8OLvNTQzEGTa/nue47m1EOn+Okf59yA8GQwxHV0dvHZXz/DTx5fy0GTa/nOe47mNE8CzrkB5slgCGvd1cnld/6N+5bVc+lJB/Dxtx7oScA5lxOeDIaoxtZdXHzLEh5btYXPnTGfi944J98hOeeKmCeDIahhRxsX3Pw4L9bv4LqFR3HWUdPyHZJzrsh5Mhhi1m5p4bwfPMbGxjZuumABJx40Kd8hOedKQKJkIGk4MNPMXshxPCWtbnsr5974V3a0dvDji1/H0TPH5jsk51yJ6LXKSUlnAEuBe2P/UZIW5TiukrO5qY3zfvAY21p2cev7jvVE4JwbVEnqH74GOBbYBmBmSwG/mjmAGlt3ccEPH2ftlhZuumABR84Yk++QnHMlJkky2GVm2zOGWbdTuj7b2d7J+3+0mOdf2cH3znstx80dn++QnHMlKMk1g2WS/hkolzQPuAx4NLdhlYb2ji7+5fYnWLJmK99a+BpOOtgvFjvn8iPJkcFHgEOBNuAOYDvw0RzGVBLMjE/+7O888mID//3OwznjyKn5Dsk5V8J6TQZm1mJmnzGzY+LrKjNrTbJwSadJekHSCklXdDN+lqQHJD0l6WFJ0/tTiEL0/T+uZNHfN/CpUw/inGNm5jsc51yJS3I30f2SxqT1j5V0X4L5yoHrgdOB+cC5kuZnTPY14FYzOwL4AvClPsResB55sYEv3/s8bz9iPz504v75Dsc55xKdJppgZttSPWa2FUhycvtYYIWZrTSzduBO4KyMaeYDD8buh7oZX3RWb2rmI3c8yUGTa/nq2Ud4a2POuSEhSTLokrT7PIakWSS7m2gasDatf10clu7vwDtj9z8CtZJedTuNpEskLZG0pKGhIcFHD03NbR1cctsSysrEjecvYESVPwDunBsakiSDzwD/J+k2SbcDfwSuHKDP/yTwJkl/A94ErAc6MycysxvMbIGZLZg4ceIAffTgSl0wXrGxiev/+WhmjBuR75Ccc263Xv+amtm9ko4GjouDPmpmmxIsez0wI61/ehyWvuwNxCMDSSOBd6Wfkiom33n4JX73TB1Xvf0Q3njAhHyH45xze0lyZAAwDNgCNALzJZ2QYJ7FwDxJcyRVAQuBvaqxkDRBUiqGK4GbE8ZTUJbX7+Drv3+Bs46ayvuP94e3nXNDT69HBpK+DJwDLAO64mAjnC7qkZl1SLoUuA8oB242s2WSvgAsMbNFwInAlySllvfh/hZkKPvS756nZlgFnzvjUL9g7JwbkpJcwXwHcJCZtfV14WZ2D3BPxrCr07p/Dvy8r8stJI+u2MSDz2/kitMPZlxNVb7Dcc65biU5TbQSqMx1IMWoq8v44j3PMW3McC58w+x8h+Occz1KcmTQAiyV9AChSgoAzOyynEVVJH799/Us29DIteccRXVleb7Dcc65HiVJBovIuPDrete6q5Ov3fcih00bxZle75BzbohLcmvpLYMRSLH54Z9Xs37bTr767iMoK/OLxs65oS3J3UTzCHUGzQeqU8PNbG4O4ypoW5rb+c5DK3jzwZN4w/7+TIFzbuhLcgH5h8B3gQ7gJOBW4PZcBlXovvXAcprbO7ji9IPzHYpzziWSJBkMN7MHAJnZGjO7Bnh7bsMqXC9vbuH2v65h4bEzmTe5Nt/hOOdcIkkuILfFp4SXx4fI1gMjcxtW4br1L6sBuOzkefkNxDnn+iDJkcHlwAhCc5evBd4LXJDLoApVS3sHdy1Zy2mHTWHK6OreZ3DOuSEiyd1Ei2NnE3BRbsMpbL9euoHG1g4u8AfMnHMFpsdkIOlaM/uopN/QTfsFZnZmTiMrMGbGLY+u5pD9RrFg1th8h+Occ32S7cjgtvj+tcEIpNA9vmoLz9ft4L/febhXRuecKzg9JgMzeyK2Y3yJmb1nEGMqSLf+ZQ2jh1dy1lGZjbk559zQl/UCspl1ArNiewSuB3XbW7l3WR3nHDOD4VVeB5FzrvAkubV0JfBnSYuA5tRAM/tGzqIqMHc8toYuM8573ax8h+Kcc/2SJBm8FF9lgD9FlaGto5M7Hn+Zkw+axMzx3q6xc64wJbm19PODEUih+t3TdWxqaud8v53UOVfAklRUNxH4N+BQ9q6o7uQcxlUwbvnLauZMqOH/eSP3zrkCluQJ5B8DzwNzgM8DqwmN3Ze8p9dt528vb+P818/yaqqdcwUtSTIYb2Y/AHaZ2SNm9j7AjwqAu5/aQGW5eNdrp+c7FOec2ydJLiDviu+vSHo7sAEYl7uQCsfi1Vs4YvoYRlV7E9HOucLW45GBpNQe7j8ljQY+AXwSuAn42CDENqS17urk6fXbOWa250XnXOHLdmSwPj5b8BOg0cyeITRu44Cla7exq9M4ZrbXQ+ScK3zZrhkcQrhQfBWwVtJ1ko7ry8IlnSbpBUkrJF3RzfiZkh6S9DdJT0l6W9/Cz58lq7cA8FqvlM45VwR6TAZmttnMvm9mJwHHEp5E/qaklyR9sbcFx3qNrgdOJ7SffK6k+RmTXQXcZWavARYC3+lnOQbd46u3ctDkWsaM8Jo6nHOFL8ndRJjZBuAHhLaQdwAfSDDbscAKM1tpZu3AncBZmYsGRsXu0YSL00NeZ5fx5JqtLPBTRM65IpE1GUiqlvRuSb8AVhBuKb0CmJpg2dOAtWn96+KwdNcA50laB9wDfKSHOC6RtETSkoaGhgQfnVvP1zXS1NbhF4+dc0Uj291EdwAvA/9EePBstpldaGb3xtpMB8K5wI/MbDrwNuC22N7yXszsBjNbYGYLJk6cOEAf3X+LV4XrBcfM8WTgnCsO2e4muhf4oJnt6Oey1wMz0vqnx2Hp3g+cBmBmf5FUDUwANvbzMwfF4jVbmTq6mmljhuc7FOecGxDZLiDfug+JAMKdSPMkzYntISwEFmVM8zLwZgBJhxDqPsr/eaAszIwlq7ewwE8ROeeKSKILyP1hZh3ApcB9wHOEu4aWSfqCpFT7yZ8ALpb0d8LzDBea2avaWx5K1m7ZSX1jm58ics4VlSTVUfSbmd1DuDCcPuzqtO5ngTfmMoaBtjg+X+APmznnikmPyUDSO7PNaGa/GPhwhr7Fq7cwqrqCAyd5Oz/OueKR7cjgjPg+CXgD8GDsPwl4FCjZZLBg9jivsto5V1R6TAZmdhGApN8D883sldi/H/CjQYluiNnc1MZLDc1eZbVzrugkuYA8I5UIonpgZo7iGdKWrNkKwLF+J5FzrsgkuYD8gKT7CHf7AJwD/CF3IQ1dS1ZvoaqijMOnj853KM45N6B6TQZmdqmkfwROiINuMLNf5jasoWnx6q0cOX00wyrK8x2Kc84NqKS3lj4J7DCzP0gaIal2Hx9IKzgt7R08s347l5wwN9+hOOfcgOv1moGki4GfA9+Pg6YBv8phTEPS0rXb6Ogyr5zOOVeUklxA/jDhwbBGADNbTrjdtKQsWb0VCY72xmycc0UoSTJoi+0RACCpgtAOQUl5Ys1WDpxUy+jhlb1P7JxzBSZJMnhE0r8DwyW9FfgZ8JvchjX0LK/fwSH7+VPHzrnilCQZXEGoSfRp4IOEuoauymVQQ82O1l1s2N7KvMmeDJxzxSnJraVdwI3xVZJeamgG4IBJI/MciXPO5UavyUDSGwnNU86K0wswMyuZeyyX14e7aOd5MnDOFakkzxn8APgY8AQwUM1dFpQVG5uoKi9j5rgR+Q7FOedyIkky2G5mv8t5JEPY8o1NzJ1YQ0V5ztoCcs65vEqSDB6S9FVCldVtqYFm9mTOohpilm/cwZHTx+Q7DOecy5kkyeB18X1B2jADTh74cIaelvYO1m3dydlHz8h3KM45lzNJ7iY6aTACGapWNjRjBvMm+8Vj51zxytbs5Xlmdrukj3c33sy+kbuwho4VG5sAv5PIOVfcsh0Z1MT3kn7SavnGHVSUiVnja3qf2DnnClS2Zi+/H98/P3jhDD3L65uYPaGGqgq/k8g5V7ySPHRWDbwfOBSoTg03s/flMK4hY8XGJg6aUtIHR865EpDk7+5twBTgVOARYDqQqGEbSadJekHSCklXdDP+m5KWxteLkrb1Ifaca+voZPXmZr9e4JwrekluLT3AzN4t6Swzu0XSHcCfeptJUjlwPfBWYB2wWNIiM3s2NY2ZfSxt+o8Ar+lzCXJo1aZmugwO8ArqnHNFLsmRwa74vk3SYcBokjVucyywwsxWxvYQ7gTOyjL9ucBPEix30Cyv9zuJnHOlIUkyuEHSWOCzwCLgWeArCeabBqxN618Xh72KpFnAHODBHsZfImmJpCUNDQ0JPnpgLN/YRJlgzgS/k8g5V9ySPHR2U+x8BMhVTaULgZ+bWbcV4ZnZDcANAAsWLBi0VtZWbNzBrPE1VFeWD9ZHOudcXmR76Kzbh81SEjx0th5Ir8NhehzWnYWEtpaHlOX1Td6GgXOuJGQ7TVTby6s3i4F5kuZIqiLs8BdlTiTpYGAs8Je+hZ5buzq7WLXJ7yRyzpWGbA+d7dPDZmbWIelS4D6gHLjZzJZJ+gKwxMxSiWEhcKeZDdrpnyTWbG6mo8u8TiLnXElI8tDZXOA64DhCbaV/AT5mZit7m9fM7iG0mZw+7OqM/mv6EO+g2XMnkd9W6pwrfknuJroDuAvYD5gK/IwhdgtoLiyPFdTNneh3Ejnnil+SZDDCzG4zs474up20aimK1fKNTUwfO5wRVUmey3POucKWZE/3u1iVxJ2E00TnAPdIGgdgZltyGF/eLK/f4RePnXMlI0ky+Kf4/sGM4QsJySFXzx7kTUdnFys3NXPCgRPzHYpzzg2KJA+dzRmMQIaStVt30t7R5c8YOOdKRq/XDCT9R6x0LtU/StIPcxtWfi2vD5Wy+mki51ypSHIBuQJ4XNIRkt5KeJjsidyGlV+pO4n8yMA5VyqSnCa6UtIfgMeArcAJZrYi55Hl0YqNTew3upra6sp8h+Kcc4MiyWmiE4BvAV8AHga+LWlqjuPKq+Ubd/hRgXOupCS5m+hrwLtTjdJIeiehqumDcxlYvnR1GS9tbGbhsePyHYpzzg2aJMng9elVS5vZLyQ9ksOY8qp+Rys7d3Uyd6IfGTjnSkePp4kkXQtgZp2SLs8Y/fVcBpVPqzY1AzBnvFdD4ZwrHdmuGZyQ1n1BxrgjchDLkJBKBrMnjMhzJM45N3iyJQP10F3UVm9qpqqijKmjh+c7FOecGzTZrhmUxbaPy9K6U0mhaNuBXLWphdnjR1BWVjL5zznnsiaD0YSHy1J7xSfTxg2phmgG0urNzcyd4NcLnHOlJVtLZ7MHMY4hobPLeHlzC28+eFK+Q3HOuUGVpDqKkrFh207aO7uY40cGzrkS48kgzZ47iTwZOOdKiyeDNKs3x2cMPBk450pMomQg6XhJF8XuiZKKso2DlQ3NjKgqZ1LtsHyH4pxzgypJRXWfAz4NXBkHVQK35zKofFm9uZnZ42uQ/LZS51xpSXJk8I/AmUAzgJltAGpzGVS+rN7U7KeInHMlKUkyaDczIz5bICnx3lLSaZJekLRC0hU9TPNPkp6VtEzSHUmXPdB2dXaxdutOr4bCOVeSktRaepek7wNjJF0MvA+4sbeZYlOZ1wNvBdYBiyUtSlWFHaeZRzj99EYz2yopbzf4r93SQmeXMdsrqHPOlaAkLZ19LTZ32QgcBFxtZvcnWPaxwAozWwkg6U7gLODZtGkuBq43s63xszb2Mf4Bk7qTaO5ETwbOudLTazKQ9HHgpwkTQLppwNq0/nXA6zKmOTB+xp8J9R1dY2b3dhPDJcAlADNnzuxjGMms2tQC4EcGzrmSlOSaQS3we0l/knSppMkD+PkVwDzgROBc4EZJYzInMrMbzGyBmS2YOHHiAH78Hqs3NVNbXcG4mqqcLN8554ayXpOBmX3ezA4FPgzsBzwi6Q8Jlr0emJHWPz0OS7cOWGRmu8xsFfAiITkMulWbQgV1flupc64U9eUJ5I1AHbAZSHKhdzEwT9IcSVXAQmBRxjS/IhwVIGkC4bTRyj7ENGBWbWr2aiiccyUryUNnH5L0MPAAMB642Mx6benMzDqAS4H7gOeAu8xsmaQvSDozTnYfsFnSs8BDwKfMbHP/itJ/rbs62bB9p18vcM6VrCS3ls4APmpmS/u6cDO7B7gnY9jVad0GfDy+8mbtlhbMvE4i51zp6jEZSBplZo3AV2P/uPTxZrYlx7ENmpWbvII651xpy3ZkcAfwD4TWzoy920E2YG4O4xpUq73qaudcicvW0tk/xPeirKE03erNzYyrqWL08Mp8h+Kcc3mR5ALyA0mGFbJVm5qZPd7rJHLOla5s1wyqgRHABElj2XOaaBTh6eKisWpTM8cfkJuH2ZxzrhBku2bwQeCjwFTCdYNUMmgE/ie3YQ2elvYO6hvbmOO1lTrnSli2awbXAddJ+oiZfXsQYxpUq1N1EvnFY+dcCUtSa+m3JR0GzAeq04bfmsvABkuqtlJ/4Mw5V8qS1Fr6OUKVEfMJD5CdDvwfUBTJYJU/Y+Ccc4nqJjobeDNQZ2YXAUcCo3Ma1SBatamZSbXDqBmW5GFs55wrTkmSwU4z6wI6JI0iVFg3o5d5CsZqr6DOOecSJYMlsY2BGwl3FT0J/CWXQQ2m1ZubmePXC5xzJS7JBeQPxc7vSboXGGVmT+U2rMHR2LqLTU3tzPGmLp1zJS7bQ2dHZxtnZk/mJqTB8/LmVFOX/oyBc660ZTsy+HqWcQacPMCxDLoN23YCMHXM8DxH4pxz+ZXtobOTBjOQfKjf0QbAlFHVvUzpnHPFLclzBud3N7wYHjqr395KeZkYP3JYvkNxzrm8SnJz/TFp3dWEZw6epAgeOqtrbGXiyGGUl6n3iZ1zrogluZvoI+n98TbTO3MV0GCqb2xl8mg/ReScc0meM8jUDBRFgzf1ja1MGeWniJxzLsk1g98Q7h6CkDzmA3flMqjBUre9lePmjs93GM45l3dJrhl8La27A1hjZutyFM+g2dneSWNrB5P9TiLnnEt0zeARgFgvUUXsHmdmW3IcW07VN7YCflupc85BsjaQL5FUBzwFLCHUT7QkycIlnSbpBUkrJF3RzfgLJTVIWhpfH+hrAfqrLiYDPzJwzrlkp4k+BRxmZpv6smBJ5cD1wFuBdcBiSYvM7NmMSX9qZpf2ZdkDYfeRwWi/gOycc0nuJnoJaOnHso8FVpjZSjNrJ9yOelY/lpMTddv9yMA551KSHBlcCTwq6TGgLTXQzC7rZb5pwNq0/nXA67qZ7l2STgBeBD5mZmszJ5B0CXAJwMyZMxOE3Lv6xjZqqsqpra4ckOU551whS3Jk8H3gQeCvhOsFqddA+A0w28yOAO4HbuluIjO7wcwWmNmCiRMnDsgH1ze2+lGBc85FSY4MKs3s4/1Y9nr2bhFtehy2m5ltTuu9CfhKPz6nX+o8GTjn3G5Jjgx+F+8o2k/SuNQrwXyLgXmS5kiqAhYCi9InkLRfWu+ZwHOJI99H9Y2tTPGqKJxzDkh2ZHBufL8ybZgBc7PNZGYdki4F7gPKgZvNbJmkLwBLzGwRcJmkMwkPs20BLuxj/P1iZmxsbGOSV0XhnHNAsofO+l0PkZndA9yTMezqtO4r2TvJDIotze20d3b5A2fOOReVZHsGdf70sXPO7aUk2zPY2BjukPXqq51zLijJ9gy8KgrnnNtbSbZnULe9FQkm1foFZOecgxJtz6C+sZXxNcOoLO9PLnTOueJTku0ZhKeP/ajAOedSekwGkg4AJqfaM0gb/kZJw8zspZxHlyN1jW1M9YvHzjm3W7bzJNcCjd0Mb4zjClZ9Y6vfSeScc2myJYPJZvZ05sA4bHbOIsqxto5OtjS3+zMGzjmXJlsyGJNl3PABjmPQ7H7GwK8ZOOfcbtmSwRJJF2cOjE1TDlQV1oOu3p8xcM65V8l2N9FHgV9Keg97dv4LgCrgH3McV87srorCrxk459xuPSYDM6sH3iDpJOCwOPi3ZvbgoESWI/Wp00S1ngyccy4lSXUUDwEPDUIsg6K+sZWqijLGjPDmLp1zLqXkHsGt297KlFHVSMp3KM45N2SUXDKob2z120qdcy5DSSYDb+HMOef2VlLJwMyo8yMD55x7lZJKBo07O2jd1eW3lTrnXIaSSgb1O8IzBpP8yMA55/ZSUsmgbru3feycc90prWTQ6MnAOee6U1LJYGNj6jSR303knHPpcpoMJJ0m6QVJKyRdkWW6d0kySQtyGU9dYytjRlRSXVmey49xzrmCk7NkIKkcuB44ndBu8rmS5nczXS1wOfBYrmJJqdve5qeInHOuG7k8MjgWWGFmK82sHbgTOKub6f4D+DLQmsNYgFTbx54MnHMuUy6TwTRgbVr/ujhsN0lHAzPM7LfZFiTpEklLJC1paGjod0AhGfj1Auecy5S3C8iSyoBvAJ/obVozu8HMFpjZgokTJ/br8zo6u9jU5KeJnHOuO7lMBuuBGWn90+OwlFpCOwkPS1oNHAcsytVF5IamNroMJvvTx8459yq5TAaLgXmS5kiqAhYCi1IjzWy7mU0ws9lmNhv4K3CmmS3JRTCpRm38yMA5514tZ8nAzDqAS4H7gOeAu8xsmaQvSDozV5/bk9TTx34B2TnnXq3Xls72hZndA9yTMezqHqY9MZex1Dd6MnDOuZ6UzBPI+42u5pT5kxlfU5XvUJxzbsjJ6ZHBUHLKoVM45dAp+Q7DOeeGpJI5MnDOOdczTwbOOec8GTjnnPNk4JxzDk8Gzjnn8GTgnHMOTwbOOefwZOCccw6QmeU7hj6R1ACs6efsE4BNAxjOUFBsZSq28kDxlanYygPFV6buyjPLzHpsA6DgksG+kLTEzHLazvJgK7YyFVt5oPjKVGzlgeIrU3/K46eJnHPOeTJwzjlXesnghnwHkAPFVqZiKw8UX5mKrTxQfGXqc3lK6pqBc8657pXakYFzzrlueDJwzjlXOslA0mmSXpC0QtIV+Y6nPyTdLGmjpGfSho2TdL+k5fF9bD5j7AtJMyQ9JOlZScskXR6HF2SZJFVLelzS32N5Ph+Hz5H0WNz2fiqpoJrbk1Qu6W+S7o79hV6e1ZKelrRU0pI4rCC3uRRJYyT9XNLzkp6T9Pq+lqkkkoGkcuB64HRgPnCupPn5japffgScljHsCuABM5sHPBD7C0UH8Akzmw8cB3w4fi+FWqY24GQzOxI4CjhN0nHAl4FvmtkBwFbg/fkLsV8uB55L6y/08gCcZGZHpd2LX6jbXMp1wL1mdjBwJOH76luZzKzoX8DrgfvS+q8Ersx3XP0sy2zgmbT+F4D9Yvd+wAv5jnEfyvZr4K3FUCZgBPAk8DrCk6AVcfhe2+JQfwHT447kZOBuQIVcnhjzamBCxrCC3eaA0cAq4g1B/S1TSRwZANOAtWn96+KwYjDZzF6J3XXA5HwG01+SZgOvAR6jgMsUT6ksBTYC9wMvAdvMrCNOUmjb3rXAvwFdsX88hV0eAAN+L+kJSZfEYQW7zQFzgAbgh/F03k2SauhjmUolGZQEC38BCu5eYUkjgf8FPmpmjenjCq1MZtZpZkcR/lEfCxyc34j6T9I/ABvN7Il8xzLAjjezowmnjT8s6YT0kYW2zQEVwNHAd83sNUAzGaeEkpSpVJLBemBGWv/0OKwY1EvaDyC+b8xzPH0iqZKQCH5sZr+Igwu6TABmtg14iHAaZYykijiqkLa9NwJnSloN3Ek4VXQdhVseAMxsfXzfCPySkLQLeZtbB6wzs8di/88JyaFPZSqVZLAYmBfvgqgCFgKL8hzTQFkEXBC7LyCcdy8IkgT8AHjOzL6RNqogyyRpoqQxsXs44frHc4SkcHacrGDKY2ZXmtl0M5tN+M08aGbvoUDLAyCpRlJtqhs4BXiGAt3mAMysDlgr6aA46M3As/S1TPm++DGIF1neBrxIOIf7mXzH088y/AR4BdhF+DfwfsI53AeA5cAfgHH5jrMP5TmecOj6FLA0vt5WqGUCjgD+FsvzDHB1HD4XeBxYAfwMGJbvWPtRthOBuwu9PDH2v8fXstS+oFC3ubRyHQUsidver4CxfS2TV0fhnHOuZE4TOeecy8KTgXPOOU8GzjnnPBk455zDk4Fzzjk8GbhBIMkkfT2t/5OSrhmgZf9I0tm9T7nPn/PuWBvkQ92MO1DSPbF2yCcl3SWpkKozeBVJ7yjQyhxdP3kycIOhDXinpAn5DiRd2lO0SbwfuNjMTspYRjXwW0JVAPMsVHPwHWDiwEWaF+8g1PDrSoQnAzcYOghtsn4sc0TmP3tJTfH9REmPSPq1pJWS/lvSe2J7AU9L2j9tMW+RtETSi7E+nVSFcV+VtFjSU5I+mLbcP0laRHhKMzOec+Pyn5H05TjsasIDcj+Q9NWMWf4Z+IuZ/SY1wMweNrNnFNo3+GFc3t8knRSXd6GkX8U65ldLulTSx+M0f5U0Lk73sKTrYr37z0g6Ng4fF+d/Kk5/RBx+jUKbFw/HdXZZWrnOi+tuqaTvx2rdkdQk6YsKbTD8VdJkSW8AzgS+GqffX9JlCu1OPCXpziRfuisw+X5yzl/F/wKagFGEqoNHA58EronjfgScnT5tfD8R2EaoencYof6bz8dxlwPXps1/L+GPzTzCk9nVwCXAVXGaYYSnM+fE5TYDc7qJcyrwMuFffQXwIPCOOO5hYEE383wDuLyHcn8CuDl2HxyXXQ1cSHh6tzZ+1nbgX+J03yRU2Jf6zBtj9wnEqsuBbwOfi90nA0tj9zXAo7G8E4DNQCVwCPAboDJO9x3g/NhtwBmx+ytp6yzze9lAfNIYGJPvbcpfA//yIwM3KCzURnorcFlv06ZZbGavmFkboRqR38fhTxPadUi5y8y6zGw5sJKw4z0FOF+hOunHCI/mz4vTP25mq7r5vGOAh82swUIVzT8m7IT763jgdgAzex5YAxwYxz1kZjvMrIGQDFJHFpll+0mc/4/AqFj30fHAbXH4g8B4SaPi9L81szYz20SomGwyoa6a1wKL4/p4M6FaBoB2QjsFAE9kfHa6p4AfSzqPcKTnikxfzpk6t6+uJTT48sO0YR3E05WSyoD0JhTb0rq70vq72HvbzaxTxQiNsHzEzO5LHyHpRMKRwUBZBrypH/PtS9mSLrczLkvALWZ2ZTfT7zIzy5i+O28nJMYzgM9IOtz2tGngioAfGbhBY2ZbgLvYu5nE1YR/rRDOU1f2Y9HvllQWryPMJbTwdB/wrwpVZKfu+KnpZTmPA2+SNCGeUz8XeKSXee4A3iDp7akBkk6QdBjwJ+A9qc8HZsbY+uKcOP/xwHYz256x3BOBTZbRDkSGB4CzJU2K84yTNKuXz91BOI2VStIzzOwh4NOEU30j+1gON8T5kYEbbF8HLk3rvxH4taS/E8799+df+8uEHfkowrn3Vkk3EU55PClJhJag3pFtIWb2iqQrCFU0i3DKJWu1v2a2M160vlbStYQaZZ8iXNf4DvBdSU8TjoAuNLO2EE5irZL+RkiS74vDrgFulvQU0MKeaop7ivFZSVcRWvcqizF+mHDaqid3AjfGi9ALCRfPRxPWy7cstNfgiojXWurcECXpYeCTZrYk37G44ueniZxzzvmRgXPOOT8ycM45hycD55xzeDJwzjmHJwPnnHN4MnDOOQf8fyr4KfSjpIJNAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# PCA to find the number of components\n",
    "pca = PCA().fit(X)\n",
    "plt.plot(np.cumsum(pca.explained_variance_ratio_))\n",
    "plt.xlabel('Number of Components')\n",
    "plt.ylabel('Cumulative Explained Variance')\n",
    "plt.title('PCA Number of Components for Cumulative Variance')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "94a6dbe0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:18:57.518518Z",
     "iopub.status.busy": "2021-11-28T18:18:57.517369Z",
     "iopub.status.idle": "2021-11-28T18:18:57.594707Z",
     "shell.execute_reply": "2021-11-28T18:18:57.595473Z",
     "shell.execute_reply.started": "2021-11-28T04:03:29.728932Z"
    },
    "papermill": {
     "duration": 0.125131,
     "end_time": "2021-11-28T18:18:57.595735",
     "exception": false,
     "start_time": "2021-11-28T18:18:57.470604",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PCA(n_components=10)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# PCA\n",
    "pca = PCA(n_components = 10)\n",
    "pca.fit(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ef9e3584",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:18:57.742092Z",
     "iopub.status.busy": "2021-11-28T18:18:57.739951Z",
     "iopub.status.idle": "2021-11-28T18:18:57.749277Z",
     "shell.execute_reply": "2021-11-28T18:18:57.750051Z",
     "shell.execute_reply.started": "2021-11-28T04:03:30.495209Z"
    },
    "papermill": {
     "duration": 0.074243,
     "end_time": "2021-11-28T18:18:57.750286",
     "exception": false,
     "start_time": "2021-11-28T18:18:57.676043",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 3.67381366e-01  6.48011123e-03 -1.72557404e-01  1.69532507e-01\n",
      "   3.58311028e-02  2.89989148e-01  3.52355142e-02  1.32550715e-01\n",
      "   7.03404149e-02  2.75029685e-01  3.63843075e-01  3.50326828e-01\n",
      "   3.64746062e-01  3.36293883e-01  2.93484097e-01  6.96012643e-02\n",
      "   1.21449009e-02  5.71420514e-02 -1.38888217e-01 -1.25546806e-02\n",
      "  -9.65928325e-03 -2.94122676e-02 -9.96684080e-03 -5.68364626e-03\n",
      "  -1.37846004e-02  0.00000000e+00  4.54505940e-05 -1.10209039e-04\n",
      "  -5.15725586e-02 -1.90922332e-03  4.74502465e-03  5.57192688e-04\n",
      "  -5.40853308e-03  0.00000000e+00 -1.24595498e-03 -1.20495364e-02\n",
      "   9.81126059e-04  8.14759410e-04  1.56442667e-03  2.28584315e-04\n",
      "   7.82317757e-03  1.41535920e-02  4.11935188e-03  2.71481491e-05\n",
      "   9.13818155e-04  4.95423787e-04 -2.30240423e-05  2.65219679e-02\n",
      "   9.16380043e-04  4.21596687e-03  1.22205015e-02  7.86182307e-03\n",
      "   3.82776229e-04  2.93701152e-03  3.67005730e-04  1.16873593e-03\n",
      "   2.45379116e-02  1.73488510e-02  1.84323512e-02]\n",
      " [ 4.77286397e-02 -4.92572888e-01  1.40776570e-01 -4.42701058e-03\n",
      "   2.86373385e-02 -5.01625298e-02  4.80666241e-01 -3.56959101e-01\n",
      "  -5.91590887e-01  5.94789839e-02  5.00324671e-02  6.07007233e-02\n",
      "   4.38445955e-02 -9.19698248e-04  5.59776164e-02  5.50254542e-02\n",
      "  -3.77195671e-05 -2.54792301e-02 -2.95085045e-02  1.93636521e-03\n",
      "  -4.26822262e-03  3.10964876e-02 -6.25116294e-03 -3.81676638e-04\n",
      "  -4.18742514e-03  0.00000000e+00 -1.27508174e-05  7.84620655e-05\n",
      "  -3.90714897e-02 -1.30199273e-03  1.51670872e-03  1.97130590e-03\n",
      "  -2.62502125e-03  0.00000000e+00 -2.43524422e-03 -1.30678455e-02\n",
      "   8.26480896e-04 -8.27761591e-04  7.20367321e-05 -4.49131391e-04\n",
      "  -1.96487960e-03 -9.85887559e-03  6.65393252e-04  6.58184952e-05\n",
      "  -2.38254490e-05 -3.66720839e-04 -6.54277397e-04  1.46207252e-02\n",
      "   2.94508820e-02 -3.19991392e-03 -3.98586757e-03 -4.53587760e-04\n",
      "  -4.28911314e-04  6.08966534e-04 -3.02575460e-05  1.38364916e-03\n",
      "  -3.90479856e-03  1.21006976e-02  3.35766063e-03]\n",
      " [ 8.94192539e-02  1.36059823e-01  4.22193198e-01  4.80760453e-01\n",
      "   6.09865356e-01 -6.30062828e-02 -2.56744049e-01 -2.91799363e-01\n",
      "  -1.21938681e-02 -1.01904138e-01  4.26426303e-02  1.08366879e-01\n",
      "   1.63904355e-06 -4.40299668e-02 -9.62760589e-03 -5.79840898e-02\n",
      "   8.46049926e-03  3.81894719e-02  1.13341187e-02 -3.74178869e-03\n",
      "  -7.96021624e-03  4.27029121e-03 -7.67665554e-03  3.13012980e-03\n",
      "   1.62051153e-03 -0.00000000e+00 -7.05324135e-05 -3.71677163e-04\n",
      "   4.58614330e-02 -5.96937727e-03 -8.13122134e-03  1.29166028e-02\n",
      "  -8.23912687e-03 -0.00000000e+00 -6.40298832e-03 -2.89956062e-02\n",
      "  -3.98926352e-03 -2.09792405e-03 -6.44849557e-03 -5.72204896e-04\n",
      "  -5.97538659e-03 -2.73492614e-02  5.85395178e-03 -2.27245870e-05\n",
      "  -1.01892897e-03  1.17613013e-03  9.29381507e-04 -1.84916168e-02\n",
      "   2.73444142e-03  1.51423008e-03  2.92325398e-03  1.62998029e-02\n",
      "   7.09750437e-04 -1.63979078e-03  5.65260480e-04 -2.53938486e-04\n",
      "   1.40801226e-03  1.68796033e-02  2.66259391e-02]\n",
      " [-2.45740649e-01  2.65698451e-02  1.41433999e-01 -2.25993121e-02\n",
      "   5.27803815e-02  1.90879901e-01 -1.16989226e-01 -1.41918518e-01\n",
      "  -2.57320687e-03  4.89002741e-01 -2.52860080e-01 -3.00877380e-01\n",
      "  -2.55311905e-01  3.65933575e-01  4.62732193e-01  7.40052112e-02\n",
      "  -3.02669965e-02 -1.31837379e-01  8.80991647e-02  6.67342939e-03\n",
      "  -3.61797501e-03  2.01706227e-02 -2.26346726e-03  4.80335916e-03\n",
      "   4.75425342e-03  0.00000000e+00  1.49535853e-04 -2.37637404e-04\n",
      "   4.11829242e-02 -6.54572294e-03  1.59846988e-02 -9.52249845e-03\n",
      "   1.32545412e-04  0.00000000e+00  3.06568108e-03  6.66168642e-03\n",
      "   8.90800612e-03 -2.27727868e-04  4.53526656e-03 -1.37323435e-03\n",
      "  -7.36360105e-03 -1.27598608e-02 -1.77998616e-03  1.29792113e-05\n",
      "   7.52701302e-04 -1.05723115e-04  1.47890180e-04  4.18953383e-02\n",
      "  -2.57086554e-03 -1.21402378e-02 -2.56940178e-02 -6.20091479e-03\n",
      "  -6.33746068e-04 -5.94921159e-03 -6.05878813e-04 -3.45941255e-04\n",
      "  -2.55681101e-02 -2.53909401e-02 -8.93361975e-03]\n",
      " [ 1.21629628e-01  1.52924736e-01  4.34632616e-01 -4.58390570e-01\n",
      "  -3.71043566e-01  9.05473314e-02 -3.48988791e-01 -4.41153360e-01\n",
      "  -1.43091477e-02  8.47910579e-03  1.74211276e-01  1.16927418e-01\n",
      "   1.77304247e-01  1.53677867e-02 -7.54867811e-02  5.91607196e-02\n",
      "   1.02705127e-02 -1.26979332e-02 -5.67332990e-02 -6.05625974e-03\n",
      "  -3.74696232e-02 -2.80430095e-02 -1.87119666e-02  2.43392678e-04\n",
      "  -4.03521897e-02 -0.00000000e+00 -4.08336034e-05 -3.05568741e-04\n",
      "   6.72158596e-02 -1.59862206e-02 -9.15091291e-03  4.50855791e-03\n",
      "  -2.57295156e-03 -0.00000000e+00 -1.36742344e-03 -1.28865971e-02\n",
      "  -3.52359943e-03  1.08872282e-03  1.63178849e-03  5.10862943e-04\n",
      "   1.00103677e-02  1.56960440e-02  8.74572672e-03  1.02981378e-04\n",
      "  -1.97595607e-03 -6.05202140e-04  1.57825321e-04  1.77248896e-02\n",
      "   1.87693952e-02 -2.65647820e-03 -1.59545306e-02  1.25151570e-02\n",
      "  -1.98383251e-03  3.35024694e-03 -8.02211495e-04  2.83673500e-03\n",
      "   7.83360669e-03  3.11666441e-02 -3.66343703e-03]\n",
      " [-3.15596430e-02  2.21364992e-02  7.21814678e-02 -1.60038200e-02\n",
      "   1.02756759e-01  7.62857741e-01  1.15706335e-01  4.75379808e-02\n",
      "  -8.72419819e-02 -3.62011659e-01 -3.11052222e-02 -1.66306780e-01\n",
      "  -4.69628051e-02  2.83642402e-01 -3.48532093e-01  2.44007269e-02\n",
      "  -5.60422806e-02 -3.02202672e-02  6.18618209e-02  7.06364534e-03\n",
      "   4.12418967e-03  4.55173617e-02 -3.28301812e-03 -1.04507013e-03\n",
      "   1.06588773e-03 -0.00000000e+00  2.37653431e-05 -4.37248269e-04\n",
      "  -1.30673463e-02 -1.69149018e-02 -8.51324895e-03  8.43245345e-03\n",
      "   2.17678500e-03 -0.00000000e+00 -1.76804811e-03 -1.80953607e-03\n",
      "  -1.46616765e-02  2.62704590e-03 -2.17328866e-03 -6.31025611e-04\n",
      "   5.00989704e-03  2.10911664e-03 -1.02700581e-02 -2.17513357e-04\n",
      "  -9.96837695e-04  2.85313276e-04 -5.77896128e-04  3.20670791e-02\n",
      "  -1.72572042e-02 -2.32166094e-03 -1.82333548e-02 -8.36908856e-03\n",
      "   9.73210203e-04  2.94678172e-03  8.67574904e-04  1.55984909e-03\n",
      "   8.51216819e-03  3.07944148e-03 -5.89354353e-03]\n",
      " [-1.34415228e-02 -6.82325294e-01 -3.23410404e-01  1.41953518e-01\n",
      "  -5.88772343e-02  9.53738556e-02 -4.28451053e-01 -2.97089399e-01\n",
      "   2.68613548e-01 -8.97976812e-02 -2.87903488e-02 -3.11430726e-02\n",
      "  -4.84379252e-03  2.67336362e-02 -5.98467517e-02 -1.72493537e-02\n",
      "  -8.32003923e-03  2.04637562e-02  5.10563675e-03 -8.72973254e-03\n",
      "   5.10089432e-03 -1.08000819e-01 -5.16330665e-02  6.01067081e-03\n",
      "   5.51247547e-03  0.00000000e+00 -2.69881473e-07  8.23059930e-04\n",
      "   1.39988833e-01  1.55511356e-02  4.78368810e-03 -2.51444188e-02\n",
      "   5.52564012e-03  0.00000000e+00 -1.46602796e-03  1.17887092e-02\n",
      "   1.89146661e-04  1.11607984e-03  3.92401164e-03 -3.37473506e-04\n",
      "  -1.85830460e-04  2.93231318e-03  5.94689381e-03 -2.11904673e-04\n",
      "  -5.33799091e-04 -3.99755501e-04  3.52394330e-04  1.33028148e-02\n",
      "  -4.62095755e-02  1.06088732e-02  1.38756449e-02 -4.65310846e-03\n",
      "   3.38224562e-04 -1.54136661e-03 -1.51424555e-04  2.26735660e-03\n",
      "   5.14509075e-04 -7.33921244e-03  6.08441668e-03]\n",
      " [-2.63452797e-02 -3.42518240e-01  4.22940866e-01 -1.72084644e-01\n",
      "   4.75830035e-02  3.20992640e-02 -1.63636162e-02  3.55776820e-01\n",
      "   1.09557473e-01  8.17566787e-02 -9.38351136e-03 -3.42682357e-02\n",
      "  -3.35627534e-02  4.48355181e-02  4.66759382e-02 -3.48733484e-01\n",
      "  -4.50011430e-02  5.57285159e-01 -1.63550532e-01  9.39624628e-04\n",
      "   1.66532100e-02  2.92476252e-02  1.32494075e-01 -7.77426366e-03\n",
      "  -4.45772411e-02  0.00000000e+00 -6.48663556e-05 -1.13395127e-03\n",
      "  -2.16477223e-02  1.31625418e-02 -1.66763280e-02  6.39174457e-02\n",
      "  -8.56754659e-03  0.00000000e+00 -2.26285718e-03  1.96371225e-03\n",
      "  -1.38808922e-04  1.90037472e-05  1.28753660e-03  3.86313998e-04\n",
      "  -2.01586602e-02 -2.17221814e-02  4.55201275e-03  5.73031506e-05\n",
      "   6.21216353e-03  1.60022440e-03  1.86684887e-03 -1.36462696e-01\n",
      "  -7.01339775e-02  1.61074305e-02  2.62341028e-02  6.83980655e-02\n",
      "   2.35815741e-04  2.39901413e-04  3.40717002e-04 -1.46791885e-03\n",
      "  -2.42501787e-02  1.24179189e-02 -2.12943949e-02]\n",
      " [-6.21619270e-02  2.78883668e-01 -1.13295356e-01  4.96644049e-01\n",
      "  -5.02828013e-01  5.83236785e-02  6.08697723e-02 -2.67555988e-01\n",
      "  -1.56136440e-01 -4.62757013e-02 -1.16445811e-01 -7.78450370e-02\n",
      "   1.16456245e-02  6.08521715e-02  4.63772643e-02 -2.88054433e-01\n",
      "  -3.78871033e-02  3.96438238e-01 -7.04967014e-02 -4.68953977e-03\n",
      "   3.07371814e-02 -1.84433027e-02  4.92221786e-03  4.82049356e-04\n",
      "  -3.41559249e-02  0.00000000e+00  9.73518822e-05 -6.97678005e-04\n",
      "   6.35631565e-02  3.64739495e-02 -5.38258047e-03  1.02530015e-02\n",
      "   5.12875904e-04  0.00000000e+00 -1.20250727e-03  2.26504018e-02\n",
      "  -2.52918591e-03 -1.14123011e-03 -1.51765399e-03 -3.66779414e-04\n",
      "  -1.34050943e-02 -2.08905294e-02  1.04220057e-02  3.91631968e-05\n",
      "   6.34347553e-03  6.07378567e-04 -1.44271814e-03 -9.61462162e-02\n",
      "  -6.42575704e-02  2.43271066e-02  6.09940450e-02  1.83875101e-02\n",
      "   3.63143976e-03 -4.33561662e-03  2.41879997e-03 -2.59920775e-03\n",
      "   1.92855436e-02 -1.93144546e-02 -2.36308640e-02]\n",
      " [-1.35929325e-03 -2.20032378e-01  4.65744820e-01  4.48226161e-01\n",
      "  -4.37692971e-01 -3.26035573e-02 -2.57392506e-02  3.33336669e-01\n",
      "   9.43651480e-02 -7.79181096e-02 -4.82391226e-02  4.15572764e-03\n",
      "   6.32688064e-02 -1.61752258e-02  7.33972949e-03  2.58890726e-01\n",
      "   2.05356391e-02 -3.18625270e-01  3.91989057e-02  2.46471522e-02\n",
      "  -5.81479096e-03  4.39346781e-02  3.58961151e-02  6.79851467e-04\n",
      "  -1.18453515e-02  0.00000000e+00  2.53145552e-05  8.82252631e-04\n",
      "  -7.26623154e-02 -2.34167275e-02  1.24746089e-02 -1.64508257e-02\n",
      "  -7.98729665e-03  0.00000000e+00 -7.31590933e-03 -3.26585502e-02\n",
      "  -2.10500905e-04 -1.84608665e-03 -8.94792011e-03 -2.14415156e-03\n",
      "   1.31392434e-02 -2.10529580e-02 -7.07507950e-03  1.90737795e-04\n",
      "   5.61008352e-04  1.38418565e-03  2.80194400e-04  1.33527987e-01\n",
      "   6.41967592e-02 -1.88216695e-02 -3.25703681e-02 -2.97342414e-02\n",
      "   9.40856204e-04 -5.51093780e-03  2.94430849e-03  1.51483584e-03\n",
      "   4.03087063e-03 -1.13647598e-02 -2.38205190e-02]]\n",
      "[6.36384013 2.60777227 1.87014089 1.66010986 1.1540086  0.64375234\n",
      " 0.49234091 0.33873994 0.28881571 0.24855664]\n"
     ]
    }
   ],
   "source": [
    "# print components\n",
    "print(pca.components_)\n",
    "\n",
    "# print variances\n",
    "print(pca.explained_variance_)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd8c5a32",
   "metadata": {
    "papermill": {
     "duration": 0.040342,
     "end_time": "2021-11-28T18:18:57.834448",
     "exception": false,
     "start_time": "2021-11-28T18:18:57.794106",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c7a66eb",
   "metadata": {
    "papermill": {
     "duration": 0.044337,
     "end_time": "2021-11-28T18:18:57.920702",
     "exception": false,
     "start_time": "2021-11-28T18:18:57.876365",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Logistic Regression did not yield high accuracy with GridSearchCV or Optuna hyperparameter tuning. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3aad3fc4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:18:58.011989Z",
     "iopub.status.busy": "2021-11-28T18:18:58.010872Z",
     "iopub.status.idle": "2021-11-28T18:19:16.361931Z",
     "shell.execute_reply": "2021-11-28T18:19:16.360637Z",
     "shell.execute_reply.started": "2021-11-28T04:04:29.235301Z"
    },
    "papermill": {
     "duration": 18.398913,
     "end_time": "2021-11-28T18:19:16.362190",
     "exception": false,
     "start_time": "2021-11-28T18:18:57.963277",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7126322751322751\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "# best model \n",
    "\n",
    "lr_model = LogisticRegression(random_state = 1, \n",
    "                              penalty = 'none', \n",
    "                              max_iter = 500, \n",
    "                              solver = 'saga')\n",
    "lr_model.fit(X, y)\n",
    "print(lr_model.score(X, y))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6248d758",
   "metadata": {
    "papermill": {
     "duration": 0.060465,
     "end_time": "2021-11-28T18:19:16.496181",
     "exception": false,
     "start_time": "2021-11-28T18:19:16.435716",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "74926b3a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:19:16.582698Z",
     "iopub.status.busy": "2021-11-28T18:19:16.581735Z",
     "iopub.status.idle": "2021-11-28T18:22:19.999115Z",
     "shell.execute_reply": "2021-11-28T18:22:19.999979Z",
     "shell.execute_reply.started": "2021-11-28T04:04:59.857826Z"
    },
    "papermill": {
     "duration": 183.46387,
     "end_time": "2021-11-28T18:22:20.000146",
     "exception": false,
     "start_time": "2021-11-28T18:19:16.536276",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-11-28 18:19:16,580]\u001b[0m A new study created in memory with name: no-name-c154bab4-2b2f-4b2f-b254-c6008b742b79\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:20,844]\u001b[0m Trial 0 finished with value: 0.6993386243386243 and parameters: {'max_depth': 15, 'min_samples_leaf': 7}. Best is trial 0 with value: 0.6993386243386243.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:22,394]\u001b[0m Trial 1 finished with value: 0.6651455026455027 and parameters: {'max_depth': 40, 'min_samples_leaf': 30}. Best is trial 0 with value: 0.6993386243386243.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:23,553]\u001b[0m Trial 2 finished with value: 0.6602513227513228 and parameters: {'max_depth': 9, 'min_samples_leaf': 31}. Best is trial 0 with value: 0.6993386243386243.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:25,078]\u001b[0m Trial 3 finished with value: 0.6922619047619047 and parameters: {'max_depth': 44, 'min_samples_leaf': 14}. Best is trial 0 with value: 0.6993386243386243.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:26,474]\u001b[0m Trial 4 finished with value: 0.6613756613756614 and parameters: {'max_depth': 10, 'min_samples_leaf': 32}. Best is trial 0 with value: 0.6993386243386243.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:28,247]\u001b[0m Trial 5 finished with value: 0.7005291005291006 and parameters: {'max_depth': 38, 'min_samples_leaf': 5}. Best is trial 5 with value: 0.7005291005291006.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:29,730]\u001b[0m Trial 6 finished with value: 0.6837301587301587 and parameters: {'max_depth': 35, 'min_samples_leaf': 18}. Best is trial 5 with value: 0.7005291005291006.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:31,279]\u001b[0m Trial 7 finished with value: 0.6922619047619047 and parameters: {'max_depth': 32, 'min_samples_leaf': 14}. Best is trial 5 with value: 0.7005291005291006.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:32,541]\u001b[0m Trial 8 finished with value: 0.6716931216931216 and parameters: {'max_depth': 33, 'min_samples_leaf': 29}. Best is trial 5 with value: 0.7005291005291006.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:33,952]\u001b[0m Trial 9 finished with value: 0.6924603174603174 and parameters: {'max_depth': 14, 'min_samples_leaf': 17}. Best is trial 5 with value: 0.7005291005291006.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:35,831]\u001b[0m Trial 10 finished with value: 0.7017857142857143 and parameters: {'max_depth': 49, 'min_samples_leaf': 2}. Best is trial 10 with value: 0.7017857142857143.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:38,046]\u001b[0m Trial 11 finished with value: 0.7139550264550265 and parameters: {'max_depth': 49, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:39,899]\u001b[0m Trial 12 finished with value: 0.7017857142857143 and parameters: {'max_depth': 50, 'min_samples_leaf': 2}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:41,856]\u001b[0m Trial 13 finished with value: 0.7139550264550265 and parameters: {'max_depth': 49, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:43,475]\u001b[0m Trial 14 finished with value: 0.6942460317460318 and parameters: {'max_depth': 22, 'min_samples_leaf': 9}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:44,892]\u001b[0m Trial 15 finished with value: 0.6701058201058201 and parameters: {'max_depth': 27, 'min_samples_leaf': 23}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:46,523]\u001b[0m Trial 16 finished with value: 0.6913359788359789 and parameters: {'max_depth': 45, 'min_samples_leaf': 10}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:47,106]\u001b[0m Trial 17 finished with value: 0.5849867724867724 and parameters: {'max_depth': 3, 'min_samples_leaf': 2}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:49,605]\u001b[0m Trial 18 finished with value: 0.7005291005291006 and parameters: {'max_depth': 43, 'min_samples_leaf': 5}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:51,074]\u001b[0m Trial 19 finished with value: 0.6724867724867725 and parameters: {'max_depth': 28, 'min_samples_leaf': 22}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:52,637]\u001b[0m Trial 20 finished with value: 0.6867724867724868 and parameters: {'max_depth': 21, 'min_samples_leaf': 11}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:54,597]\u001b[0m Trial 21 finished with value: 0.7139550264550265 and parameters: {'max_depth': 50, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:56,502]\u001b[0m Trial 22 finished with value: 0.7139550264550265 and parameters: {'max_depth': 47, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:19:58,227]\u001b[0m Trial 23 finished with value: 0.7005291005291006 and parameters: {'max_depth': 40, 'min_samples_leaf': 5}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:00,199]\u001b[0m Trial 24 finished with value: 0.6938492063492063 and parameters: {'max_depth': 50, 'min_samples_leaf': 7}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:01,972]\u001b[0m Trial 25 finished with value: 0.694047619047619 and parameters: {'max_depth': 46, 'min_samples_leaf': 4}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:03,902]\u001b[0m Trial 26 finished with value: 0.7139550264550265 and parameters: {'max_depth': 37, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:05,565]\u001b[0m Trial 27 finished with value: 0.6938492063492063 and parameters: {'max_depth': 42, 'min_samples_leaf': 8}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:07,134]\u001b[0m Trial 28 finished with value: 0.6859788359788359 and parameters: {'max_depth': 46, 'min_samples_leaf': 12}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:08,838]\u001b[0m Trial 29 finished with value: 0.6913359788359789 and parameters: {'max_depth': 36, 'min_samples_leaf': 6}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:10,973]\u001b[0m Trial 30 finished with value: 0.7007275132275133 and parameters: {'max_depth': 30, 'min_samples_leaf': 3}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:12,922]\u001b[0m Trial 31 finished with value: 0.7139550264550265 and parameters: {'max_depth': 47, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:14,683]\u001b[0m Trial 32 finished with value: 0.694047619047619 and parameters: {'max_depth': 40, 'min_samples_leaf': 4}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:16,592]\u001b[0m Trial 33 finished with value: 0.7139550264550265 and parameters: {'max_depth': 46, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:18,263]\u001b[0m Trial 34 finished with value: 0.6938492063492063 and parameters: {'max_depth': 40, 'min_samples_leaf': 7}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:20,175]\u001b[0m Trial 35 finished with value: 0.7021164021164022 and parameters: {'max_depth': 48, 'min_samples_leaf': 3}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:22,869]\u001b[0m Trial 36 finished with value: 0.694047619047619 and parameters: {'max_depth': 43, 'min_samples_leaf': 4}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:24,570]\u001b[0m Trial 37 finished with value: 0.6938492063492063 and parameters: {'max_depth': 38, 'min_samples_leaf': 8}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:26,153]\u001b[0m Trial 38 finished with value: 0.6941137566137565 and parameters: {'max_depth': 46, 'min_samples_leaf': 13}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:27,603]\u001b[0m Trial 39 finished with value: 0.6771164021164021 and parameters: {'max_depth': 36, 'min_samples_leaf': 20}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:29,002]\u001b[0m Trial 40 finished with value: 0.6682539682539682 and parameters: {'max_depth': 42, 'min_samples_leaf': 25}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:30,957]\u001b[0m Trial 41 finished with value: 0.7139550264550265 and parameters: {'max_depth': 48, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:33,195]\u001b[0m Trial 42 finished with value: 0.7139550264550265 and parameters: {'max_depth': 47, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:35,021]\u001b[0m Trial 43 finished with value: 0.7021164021164022 and parameters: {'max_depth': 44, 'min_samples_leaf': 3}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:36,749]\u001b[0m Trial 44 finished with value: 0.6913359788359789 and parameters: {'max_depth': 38, 'min_samples_leaf': 6}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:38,283]\u001b[0m Trial 45 finished with value: 0.6882275132275132 and parameters: {'max_depth': 49, 'min_samples_leaf': 15}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:40,105]\u001b[0m Trial 46 finished with value: 0.7021164021164022 and parameters: {'max_depth': 47, 'min_samples_leaf': 3}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:42,048]\u001b[0m Trial 47 finished with value: 0.7132936507936508 and parameters: {'max_depth': 34, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:43,576]\u001b[0m Trial 48 finished with value: 0.6699735449735449 and parameters: {'max_depth': 42, 'min_samples_leaf': 27}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:45,334]\u001b[0m Trial 49 finished with value: 0.6913359788359789 and parameters: {'max_depth': 50, 'min_samples_leaf': 6}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:47,211]\u001b[0m Trial 50 finished with value: 0.7017857142857143 and parameters: {'max_depth': 45, 'min_samples_leaf': 2}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:49,120]\u001b[0m Trial 51 finished with value: 0.7139550264550265 and parameters: {'max_depth': 44, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:50,910]\u001b[0m Trial 52 finished with value: 0.6941798941798941 and parameters: {'max_depth': 23, 'min_samples_leaf': 4}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:52,881]\u001b[0m Trial 53 finished with value: 0.7017857142857143 and parameters: {'max_depth': 48, 'min_samples_leaf': 2}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:55,680]\u001b[0m Trial 54 finished with value: 0.7139550264550265 and parameters: {'max_depth': 44, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:57,477]\u001b[0m Trial 55 finished with value: 0.7021164021164022 and parameters: {'max_depth': 38, 'min_samples_leaf': 3}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:20:59,207]\u001b[0m Trial 56 finished with value: 0.7005291005291006 and parameters: {'max_depth': 44, 'min_samples_leaf': 5}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:01,148]\u001b[0m Trial 57 finished with value: 0.7139550264550265 and parameters: {'max_depth': 41, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:03,014]\u001b[0m Trial 58 finished with value: 0.7017857142857143 and parameters: {'max_depth': 48, 'min_samples_leaf': 2}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:04,642]\u001b[0m Trial 59 finished with value: 0.6935185185185185 and parameters: {'max_depth': 50, 'min_samples_leaf': 9}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:06,651]\u001b[0m Trial 60 finished with value: 0.6984788359788359 and parameters: {'max_depth': 13, 'min_samples_leaf': 5}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:08,621]\u001b[0m Trial 61 finished with value: 0.7139550264550265 and parameters: {'max_depth': 47, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:10,491]\u001b[0m Trial 62 finished with value: 0.7017857142857143 and parameters: {'max_depth': 47, 'min_samples_leaf': 2}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:12,282]\u001b[0m Trial 63 finished with value: 0.694047619047619 and parameters: {'max_depth': 31, 'min_samples_leaf': 4}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:14,130]\u001b[0m Trial 64 finished with value: 0.7021164021164022 and parameters: {'max_depth': 50, 'min_samples_leaf': 3}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:15,919]\u001b[0m Trial 65 finished with value: 0.7017857142857143 and parameters: {'max_depth': 45, 'min_samples_leaf': 2}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:18,152]\u001b[0m Trial 66 finished with value: 0.7139550264550265 and parameters: {'max_depth': 40, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:20,072]\u001b[0m Trial 67 finished with value: 0.7139550264550265 and parameters: {'max_depth': 42, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:21,724]\u001b[0m Trial 68 finished with value: 0.6960978835978835 and parameters: {'max_depth': 18, 'min_samples_leaf': 6}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:22,418]\u001b[0m Trial 69 finished with value: 0.6133597883597884 and parameters: {'max_depth': 5, 'min_samples_leaf': 4}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:24,237]\u001b[0m Trial 70 finished with value: 0.7021164021164022 and parameters: {'max_depth': 45, 'min_samples_leaf': 3}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:26,817]\u001b[0m Trial 71 finished with value: 0.7139550264550265 and parameters: {'max_depth': 41, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:29,038]\u001b[0m Trial 72 finished with value: 0.7017857142857143 and parameters: {'max_depth': 48, 'min_samples_leaf': 2}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:30,834]\u001b[0m Trial 73 finished with value: 0.694047619047619 and parameters: {'max_depth': 39, 'min_samples_leaf': 4}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:32,712]\u001b[0m Trial 74 finished with value: 0.7017857142857143 and parameters: {'max_depth': 43, 'min_samples_leaf': 2}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:34,211]\u001b[0m Trial 75 finished with value: 0.6837301587301587 and parameters: {'max_depth': 49, 'min_samples_leaf': 18}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:36,150]\u001b[0m Trial 76 finished with value: 0.7139550264550265 and parameters: {'max_depth': 47, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:37,514]\u001b[0m Trial 77 finished with value: 0.6625 and parameters: {'max_depth': 46, 'min_samples_leaf': 32}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:39,563]\u001b[0m Trial 78 finished with value: 0.7005291005291006 and parameters: {'max_depth': 45, 'min_samples_leaf': 5}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:41,436]\u001b[0m Trial 79 finished with value: 0.7021164021164022 and parameters: {'max_depth': 36, 'min_samples_leaf': 3}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:43,358]\u001b[0m Trial 80 finished with value: 0.7139550264550265 and parameters: {'max_depth': 44, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:45,339]\u001b[0m Trial 81 finished with value: 0.7139550264550265 and parameters: {'max_depth': 44, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:47,212]\u001b[0m Trial 82 finished with value: 0.7017857142857143 and parameters: {'max_depth': 43, 'min_samples_leaf': 2}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:49,022]\u001b[0m Trial 83 finished with value: 0.7021164021164022 and parameters: {'max_depth': 49, 'min_samples_leaf': 3}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:51,217]\u001b[0m Trial 84 finished with value: 0.7139550264550265 and parameters: {'max_depth': 47, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:52,987]\u001b[0m Trial 85 finished with value: 0.7005291005291006 and parameters: {'max_depth': 41, 'min_samples_leaf': 5}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:54,857]\u001b[0m Trial 86 finished with value: 0.7017857142857143 and parameters: {'max_depth': 46, 'min_samples_leaf': 2}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:56,788]\u001b[0m Trial 87 finished with value: 0.7139550264550265 and parameters: {'max_depth': 41, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:21:59,178]\u001b[0m Trial 88 finished with value: 0.694047619047619 and parameters: {'max_depth': 39, 'min_samples_leaf': 4}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:22:01,381]\u001b[0m Trial 89 finished with value: 0.7017857142857143 and parameters: {'max_depth': 37, 'min_samples_leaf': 2}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:22:03,216]\u001b[0m Trial 90 finished with value: 0.7021164021164022 and parameters: {'max_depth': 42, 'min_samples_leaf': 3}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:22:05,187]\u001b[0m Trial 91 finished with value: 0.7139550264550265 and parameters: {'max_depth': 47, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:22:07,157]\u001b[0m Trial 92 finished with value: 0.7139550264550265 and parameters: {'max_depth': 47, 'min_samples_leaf': 1}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:22:09,053]\u001b[0m Trial 93 finished with value: 0.7017857142857143 and parameters: {'max_depth': 48, 'min_samples_leaf': 2}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:22:10,899]\u001b[0m Trial 94 finished with value: 0.7021164021164022 and parameters: {'max_depth': 44, 'min_samples_leaf': 3}. Best is trial 11 with value: 0.7139550264550265.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:22:13,141]\u001b[0m Trial 95 finished with value: 0.714484126984127 and parameters: {'max_depth': 33, 'min_samples_leaf': 1}. Best is trial 95 with value: 0.714484126984127.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:22:14,940]\u001b[0m Trial 96 finished with value: 0.694047619047619 and parameters: {'max_depth': 33, 'min_samples_leaf': 4}. Best is trial 95 with value: 0.714484126984127.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:22:16,876]\u001b[0m Trial 97 finished with value: 0.7118386243386243 and parameters: {'max_depth': 28, 'min_samples_leaf': 1}. Best is trial 95 with value: 0.714484126984127.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:22:18,575]\u001b[0m Trial 98 finished with value: 0.6913359788359789 and parameters: {'max_depth': 35, 'min_samples_leaf': 6}. Best is trial 95 with value: 0.714484126984127.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:22:19,990]\u001b[0m Trial 99 finished with value: 0.6724867724867725 and parameters: {'max_depth': 39, 'min_samples_leaf': 22}. Best is trial 95 with value: 0.714484126984127.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.714484126984127\n",
      "Best hyperparameters: {'max_depth': 33, 'min_samples_leaf': 1}\n",
      "CPU times: user 5.77 s, sys: 709 ms, total: 6.48 s\n",
      "Wall time: 3min 3s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# optuna hyperparameter tuning\n",
    "def objective(trial):\n",
    "    max_depth = trial.suggest_int('max_depth', 2, 50)\n",
    "    min_samples_leaf = trial.suggest_int('min_samples_leaf', 1, 32)\n",
    "    dt_clf = DecisionTreeClassifier(random_state = 1, max_depth = max_depth, min_samples_leaf = min_samples_leaf)\n",
    "    return sklearn.model_selection.cross_val_score(dt_clf, X, y, n_jobs = -1, cv = 10).mean()\n",
    "    \n",
    "dt_study = optuna.create_study(direction='maximize')\n",
    "dt_study.optimize(objective, n_trials=100)\n",
    "dt = dt_study.best_trial\n",
    "print('Accuracy: {}'.format(dt.value))\n",
    "print(\"Best hyperparameters: {}\".format(dt.params))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e0ada82",
   "metadata": {
    "papermill": {
     "duration": 0.06486,
     "end_time": "2021-11-28T18:22:20.131325",
     "exception": false,
     "start_time": "2021-11-28T18:22:20.066465",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Best Params\n",
    "\n",
    "- Accuracy: 0.7187169312169313\n",
    "- Best hyperparameters: {'max_depth': 19, 'min_samples_leaf': 1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "06127363",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:22:20.266654Z",
     "iopub.status.busy": "2021-11-28T18:22:20.266134Z",
     "iopub.status.idle": "2021-11-28T18:22:20.544502Z",
     "shell.execute_reply": "2021-11-28T18:22:20.544878Z",
     "shell.execute_reply.started": "2021-11-28T04:09:09.584882Z"
    },
    "papermill": {
     "duration": 0.348622,
     "end_time": "2021-11-28T18:22:20.545059",
     "exception": false,
     "start_time": "2021-11-28T18:22:20.196437",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(max_depth=33, random_state=1)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dt best model\n",
    "dt_model = DecisionTreeClassifier(random_state = 1, \n",
    "                                  max_depth = dt_study.best_trial.params['max_depth'], \n",
    "                                  min_samples_leaf = dt_study.best_trial.params['min_samples_leaf'])\n",
    "dt_model.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a6443de",
   "metadata": {
    "papermill": {
     "duration": 0.066904,
     "end_time": "2021-11-28T18:22:20.678148",
     "exception": false,
     "start_time": "2021-11-28T18:22:20.611244",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "355e4ec5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:22:20.820028Z",
     "iopub.status.busy": "2021-11-28T18:22:20.819165Z",
     "iopub.status.idle": "2021-11-28T18:29:10.690998Z",
     "shell.execute_reply": "2021-11-28T18:29:10.691551Z",
     "shell.execute_reply.started": "2021-11-28T04:09:13.681665Z"
    },
    "papermill": {
     "duration": 409.946751,
     "end_time": "2021-11-28T18:29:10.691762",
     "exception": false,
     "start_time": "2021-11-28T18:22:20.745011",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-11-28 18:22:20,817]\u001b[0m A new study created in memory with name: no-name-c454d7f4-9dca-4ba1-bec6-15f2d07fbe36\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:22:41,791]\u001b[0m Trial 0 finished with value: 0.7430555555555556 and parameters: {'n_estimators': 148, 'max_depth': 41, 'min_samples_leaf': 13}. Best is trial 0 with value: 0.7430555555555556.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:22:56,757]\u001b[0m Trial 1 finished with value: 0.7444444444444445 and parameters: {'n_estimators': 107, 'max_depth': 21, 'min_samples_leaf': 13}. Best is trial 1 with value: 0.7444444444444445.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:23:16,379]\u001b[0m Trial 2 finished with value: 0.7482142857142857 and parameters: {'n_estimators': 135, 'max_depth': 21, 'min_samples_leaf': 10}. Best is trial 2 with value: 0.7482142857142857.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:23:38,925]\u001b[0m Trial 3 finished with value: 0.756084656084656 and parameters: {'n_estimators': 146, 'max_depth': 34, 'min_samples_leaf': 7}. Best is trial 3 with value: 0.756084656084656.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:23:52,320]\u001b[0m Trial 4 finished with value: 0.7326058201058202 and parameters: {'n_estimators': 100, 'max_depth': 36, 'min_samples_leaf': 18}. Best is trial 3 with value: 0.756084656084656.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:24:08,035]\u001b[0m Trial 5 finished with value: 0.7301587301587302 and parameters: {'n_estimators': 117, 'max_depth': 30, 'min_samples_leaf': 19}. Best is trial 3 with value: 0.756084656084656.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:24:30,707]\u001b[0m Trial 6 finished with value: 0.7981481481481482 and parameters: {'n_estimators': 124, 'max_depth': 30, 'min_samples_leaf': 1}. Best is trial 6 with value: 0.7981481481481482.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:24:50,002]\u001b[0m Trial 7 finished with value: 0.7626984126984127 and parameters: {'n_estimators': 116, 'max_depth': 29, 'min_samples_leaf': 5}. Best is trial 6 with value: 0.7981481481481482.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:25:07,348]\u001b[0m Trial 8 finished with value: 0.7595899470899471 and parameters: {'n_estimators': 114, 'max_depth': 42, 'min_samples_leaf': 6}. Best is trial 6 with value: 0.7981481481481482.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:25:28,019]\u001b[0m Trial 9 finished with value: 0.7390873015873016 and parameters: {'n_estimators': 148, 'max_depth': 29, 'min_samples_leaf': 15}. Best is trial 6 with value: 0.7981481481481482.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:25:52,664]\u001b[0m Trial 10 finished with value: 0.8003968253968254 and parameters: {'n_estimators': 130, 'max_depth': 49, 'min_samples_leaf': 1}. Best is trial 10 with value: 0.8003968253968254.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:26:16,832]\u001b[0m Trial 11 finished with value: 0.7995370370370369 and parameters: {'n_estimators': 129, 'max_depth': 49, 'min_samples_leaf': 1}. Best is trial 10 with value: 0.8003968253968254.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:26:38,848]\u001b[0m Trial 12 finished with value: 0.7789021164021165 and parameters: {'n_estimators': 132, 'max_depth': 50, 'min_samples_leaf': 3}. Best is trial 10 with value: 0.8003968253968254.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:27:04,452]\u001b[0m Trial 13 finished with value: 0.7982142857142855 and parameters: {'n_estimators': 135, 'max_depth': 50, 'min_samples_leaf': 1}. Best is trial 10 with value: 0.8003968253968254.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:27:23,661]\u001b[0m Trial 14 finished with value: 0.7492724867724868 and parameters: {'n_estimators': 127, 'max_depth': 45, 'min_samples_leaf': 9}. Best is trial 10 with value: 0.8003968253968254.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:27:47,068]\u001b[0m Trial 15 finished with value: 0.7783730158730159 and parameters: {'n_estimators': 141, 'max_depth': 46, 'min_samples_leaf': 3}. Best is trial 10 with value: 0.8003968253968254.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:28:07,545]\u001b[0m Trial 16 finished with value: 0.7699074074074075 and parameters: {'n_estimators': 123, 'max_depth': 45, 'min_samples_leaf': 4}. Best is trial 10 with value: 0.8003968253968254.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:28:27,844]\u001b[0m Trial 17 finished with value: 0.7539021164021164 and parameters: {'n_estimators': 131, 'max_depth': 39, 'min_samples_leaf': 8}. Best is trial 10 with value: 0.8003968253968254.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:28:50,005]\u001b[0m Trial 18 finished with value: 0.7995370370370369 and parameters: {'n_estimators': 121, 'max_depth': 48, 'min_samples_leaf': 1}. Best is trial 10 with value: 0.8003968253968254.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:29:10,683]\u001b[0m Trial 19 finished with value: 0.7778439153439154 and parameters: {'n_estimators': 120, 'max_depth': 47, 'min_samples_leaf': 3}. Best is trial 10 with value: 0.8003968253968254.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.8003968253968254\n",
      "Best hyperparameters: {'n_estimators': 130, 'max_depth': 49, 'min_samples_leaf': 1}\n",
      "CPU times: user 1.29 s, sys: 171 ms, total: 1.47 s\n",
      "Wall time: 6min 49s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# optuna hyperparameter tuning\n",
    "def objective(trial):\n",
    "    n_estimators = trial.suggest_int('n_estimators', 100, 150)\n",
    "    max_depth = trial.suggest_int('max_depth', 20, 50)\n",
    "    min_samples_leaf = trial.suggest_int('min_samples_leaf', 1, 20)\n",
    "    rf_clf = RandomForestClassifier(random_state = 1, n_estimators = n_estimators, max_depth = max_depth, min_samples_leaf = min_samples_leaf)\n",
    "    return sklearn.model_selection.cross_val_score(rf_clf, X, y, n_jobs = -1, cv = 10).mean()\n",
    "    \n",
    "rf_study = optuna.create_study(direction='maximize')\n",
    "rf_study.optimize(objective, n_trials=20)\n",
    "rf = rf_study.best_trial\n",
    "print('Accuracy: {}'.format(rf.value))\n",
    "print(\"Best hyperparameters: {}\".format(rf.params))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "229a5f09",
   "metadata": {
    "papermill": {
     "duration": 0.071313,
     "end_time": "2021-11-28T18:29:10.835230",
     "exception": false,
     "start_time": "2021-11-28T18:29:10.763917",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Best Params\n",
    "\n",
    "- Accuracy: 0.8003968253968254\n",
    "- Best hyperparameters: {'n_estimators': 130, 'max_depth': 50, 'min_samples_leaf': 1}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ce70767f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:29:10.991937Z",
     "iopub.status.busy": "2021-11-28T18:29:10.991088Z",
     "iopub.status.idle": "2021-11-28T18:29:14.371586Z",
     "shell.execute_reply": "2021-11-28T18:29:14.372063Z",
     "shell.execute_reply.started": "2021-11-28T04:16:51.564144Z"
    },
    "papermill": {
     "duration": 3.465332,
     "end_time": "2021-11-28T18:29:14.372229",
     "exception": false,
     "start_time": "2021-11-28T18:29:10.906897",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(max_depth=49, n_estimators=130, random_state=1)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# best model\n",
    "rf_model = RandomForestClassifier(random_state = 1, \n",
    "                                  n_estimators = rf_study.best_trial.params['n_estimators'], \n",
    "                                  max_depth = rf_study.best_trial.params['max_depth'], \n",
    "                                  min_samples_leaf = rf_study.best_trial.params['min_samples_leaf'])\n",
    "\n",
    "rf_model.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b46fd9ca",
   "metadata": {
    "papermill": {
     "duration": 0.071572,
     "end_time": "2021-11-28T18:29:14.516810",
     "exception": false,
     "start_time": "2021-11-28T18:29:14.445238",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Extra Tree Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "60b755f1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:29:14.666406Z",
     "iopub.status.busy": "2021-11-28T18:29:14.665533Z",
     "iopub.status.idle": "2021-11-28T18:34:27.722434Z",
     "shell.execute_reply": "2021-11-28T18:34:27.723118Z",
     "shell.execute_reply.started": "2021-11-28T04:17:02.442587Z"
    },
    "papermill": {
     "duration": 313.134338,
     "end_time": "2021-11-28T18:34:27.723323",
     "exception": false,
     "start_time": "2021-11-28T18:29:14.588985",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-11-28 18:29:14,663]\u001b[0m A new study created in memory with name: no-name-fa83311a-76aa-4bce-ae8d-b125cb48ddac\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:29:26,965]\u001b[0m Trial 0 finished with value: 0.7062169312169313 and parameters: {'max_depth': 33, 'min_samples_leaf': 17}. Best is trial 0 with value: 0.7062169312169313.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:29:42,421]\u001b[0m Trial 1 finished with value: 0.7433201058201058 and parameters: {'max_depth': 35, 'min_samples_leaf': 5}. Best is trial 1 with value: 0.7433201058201058.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:29:55,334]\u001b[0m Trial 2 finished with value: 0.7062169312169313 and parameters: {'max_depth': 36, 'min_samples_leaf': 17}. Best is trial 1 with value: 0.7433201058201058.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:30:08,649]\u001b[0m Trial 3 finished with value: 0.7083994708994709 and parameters: {'max_depth': 43, 'min_samples_leaf': 15}. Best is trial 1 with value: 0.7433201058201058.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:30:20,942]\u001b[0m Trial 4 finished with value: 0.7009920634920636 and parameters: {'max_depth': 38, 'min_samples_leaf': 18}. Best is trial 1 with value: 0.7433201058201058.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:30:33,820]\u001b[0m Trial 5 finished with value: 0.7062169312169313 and parameters: {'max_depth': 47, 'min_samples_leaf': 17}. Best is trial 1 with value: 0.7433201058201058.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:30:47,683]\u001b[0m Trial 6 finished with value: 0.7296957671957672 and parameters: {'max_depth': 48, 'min_samples_leaf': 7}. Best is trial 1 with value: 0.7433201058201058.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:31:00,069]\u001b[0m Trial 7 finished with value: 0.6993386243386244 and parameters: {'max_depth': 36, 'min_samples_leaf': 20}. Best is trial 1 with value: 0.7433201058201058.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:31:22,638]\u001b[0m Trial 8 finished with value: 0.8088624338624338 and parameters: {'max_depth': 40, 'min_samples_leaf': 1}. Best is trial 8 with value: 0.8088624338624338.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:31:35,279]\u001b[0m Trial 9 finished with value: 0.7123677248677249 and parameters: {'max_depth': 38, 'min_samples_leaf': 14}. Best is trial 8 with value: 0.8088624338624338.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:31:57,578]\u001b[0m Trial 10 finished with value: 0.8089285714285713 and parameters: {'max_depth': 30, 'min_samples_leaf': 1}. Best is trial 10 with value: 0.8089285714285713.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:32:20,084]\u001b[0m Trial 11 finished with value: 0.8089285714285713 and parameters: {'max_depth': 30, 'min_samples_leaf': 1}. Best is trial 10 with value: 0.8089285714285713.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:32:42,617]\u001b[0m Trial 12 finished with value: 0.8089285714285713 and parameters: {'max_depth': 30, 'min_samples_leaf': 1}. Best is trial 10 with value: 0.8089285714285713.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:32:57,294]\u001b[0m Trial 13 finished with value: 0.741931216931217 and parameters: {'max_depth': 31, 'min_samples_leaf': 5}. Best is trial 10 with value: 0.8089285714285713.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:33:10,286]\u001b[0m Trial 14 finished with value: 0.7194444444444443 and parameters: {'max_depth': 30, 'min_samples_leaf': 10}. Best is trial 10 with value: 0.8089285714285713.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:33:26,114]\u001b[0m Trial 15 finished with value: 0.7501322751322751 and parameters: {'max_depth': 33, 'min_samples_leaf': 4}. Best is trial 10 with value: 0.8089285714285713.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:33:39,338]\u001b[0m Trial 16 finished with value: 0.7197089947089947 and parameters: {'max_depth': 44, 'min_samples_leaf': 10}. Best is trial 10 with value: 0.8089285714285713.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:33:57,616]\u001b[0m Trial 17 finished with value: 0.7758597883597884 and parameters: {'max_depth': 33, 'min_samples_leaf': 2}. Best is trial 10 with value: 0.8089285714285713.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:34:11,393]\u001b[0m Trial 18 finished with value: 0.7256613756613757 and parameters: {'max_depth': 32, 'min_samples_leaf': 8}. Best is trial 10 with value: 0.8089285714285713.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:34:27,715]\u001b[0m Trial 19 finished with value: 0.7593915343915343 and parameters: {'max_depth': 50, 'min_samples_leaf': 3}. Best is trial 10 with value: 0.8089285714285713.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.8089285714285713\n",
      "Best hyperparameters: {'max_depth': 30, 'min_samples_leaf': 1}\n",
      "CPU times: user 1.21 s, sys: 178 ms, total: 1.39 s\n",
      "Wall time: 5min 13s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# optuna hyperparameter tuning\n",
    "def objective(trial):\n",
    "    max_depth = trial.suggest_int('max_depth', 30, 50)\n",
    "    min_samples_leaf = trial.suggest_int('min_samples_leaf', 1, 20)\n",
    "    tree_clf = ExtraTreesClassifier(random_state = 0, n_estimators = 200, max_depth = max_depth, min_samples_leaf = min_samples_leaf)\n",
    "    return sklearn.model_selection.cross_val_score(tree_clf, X, y, n_jobs = -1, cv = 10).mean()\n",
    "    \n",
    "tree_study = optuna.create_study(direction='maximize')\n",
    "tree_study.optimize(objective, n_trials=20)\n",
    "tree = tree_study.best_trial\n",
    "print('Accuracy: {}'.format(tree.value))\n",
    "print(\"Best hyperparameters: {}\".format(tree.params))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99f54ea5",
   "metadata": {
    "papermill": {
     "duration": 0.077156,
     "end_time": "2021-11-28T18:34:27.881496",
     "exception": false,
     "start_time": "2021-11-28T18:34:27.804340",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Best Params\n",
    "\n",
    "- Accuracy: 0.809457671957672\n",
    "- Best hyperparameters: {'max_depth': 33, 'min_samples_leaf': 1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5a515ac0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:34:28.121126Z",
     "iopub.status.busy": "2021-11-28T18:34:28.119177Z",
     "iopub.status.idle": "2021-11-28T18:34:31.064514Z",
     "shell.execute_reply": "2021-11-28T18:34:31.065029Z",
     "shell.execute_reply.started": "2021-11-28T04:22:30.293109Z"
    },
    "papermill": {
     "duration": 3.102056,
     "end_time": "2021-11-28T18:34:31.065173",
     "exception": false,
     "start_time": "2021-11-28T18:34:27.963117",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ExtraTreesClassifier(max_depth=30, n_estimators=200, random_state=1)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# best model\n",
    "tree_model = ExtraTreesClassifier(random_state = 1, \n",
    "                                  n_estimators = 200, \n",
    "                                  max_depth = tree_study.best_trial.params['max_depth'], \n",
    "                                  min_samples_leaf = tree_study.best_trial.params['min_samples_leaf'])\n",
    "\n",
    "tree_model.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac6bbab4",
   "metadata": {
    "papermill": {
     "duration": 0.078328,
     "end_time": "2021-11-28T18:34:31.221030",
     "exception": false,
     "start_time": "2021-11-28T18:34:31.142702",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Gradient Boosting Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e09227cc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T18:34:31.382390Z",
     "iopub.status.busy": "2021-11-28T18:34:31.381593Z",
     "iopub.status.idle": "2021-11-28T20:13:41.996217Z",
     "shell.execute_reply": "2021-11-28T20:13:41.995713Z",
     "shell.execute_reply.started": "2021-11-28T04:22:49.506991Z"
    },
    "papermill": {
     "duration": 5950.697924,
     "end_time": "2021-11-28T20:13:41.996336",
     "exception": false,
     "start_time": "2021-11-28T18:34:31.298412",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-11-28 18:34:31,380]\u001b[0m A new study created in memory with name: no-name-5dfeda37-0d19-48fa-a2a0-37ed9d6d49ac\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 18:54:23,483]\u001b[0m Trial 0 finished with value: 0.80244708994709 and parameters: {'max_depth': 15, 'min_samples_leaf': 20}. Best is trial 0 with value: 0.80244708994709.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 19:13:22,801]\u001b[0m Trial 1 finished with value: 0.8003968253968254 and parameters: {'max_depth': 14, 'min_samples_leaf': 17}. Best is trial 0 with value: 0.80244708994709.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 19:38:10,444]\u001b[0m Trial 2 finished with value: 0.8019179894179894 and parameters: {'max_depth': 20, 'min_samples_leaf': 16}. Best is trial 0 with value: 0.80244708994709.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 19:58:11,967]\u001b[0m Trial 3 finished with value: 0.7990079365079366 and parameters: {'max_depth': 15, 'min_samples_leaf': 18}. Best is trial 0 with value: 0.80244708994709.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:13:41,909]\u001b[0m Trial 4 finished with value: 0.7970899470899471 and parameters: {'max_depth': 11, 'min_samples_leaf': 20}. Best is trial 0 with value: 0.80244708994709.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.80244708994709\n",
      "Best hyperparameters: {'max_depth': 15, 'min_samples_leaf': 20}\n",
      "CPU times: user 664 ms, sys: 530 ms, total: 1.19 s\n",
      "Wall time: 1h 39min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# optuna hyperparameter tuning\n",
    "def objective(trial):\n",
    "    max_depth = trial.suggest_int('max_depth', 10, 20)\n",
    "    min_samples_leaf = trial.suggest_int('min_samples_leaf', 15, 20)\n",
    "    gradb_clf = GradientBoostingClassifier(random_state = 0, max_depth = max_depth, min_samples_leaf = min_samples_leaf)\n",
    "    return sklearn.model_selection.cross_val_score(gradb_clf, X, y, n_jobs = -1, cv = 10).mean()\n",
    "    \n",
    "gradb_study = optuna.create_study(direction='maximize')\n",
    "gradb_study.optimize(objective, n_trials = 5)\n",
    "gradb = gradb_study.best_trial\n",
    "print('Accuracy: {}'.format(gradb.value))\n",
    "print(\"Best hyperparameters: {}\".format(gradb.params))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b69ef84",
   "metadata": {
    "papermill": {
     "duration": 0.079102,
     "end_time": "2021-11-28T20:13:42.154474",
     "exception": false,
     "start_time": "2021-11-28T20:13:42.075372",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Best Params\n",
    "\n",
    "- Accuracy: 0.8049603174603176\n",
    "- Best hyperparameters: {'max_depth': 19, 'min_samples_leaf': 16}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4eeb58a4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T20:13:42.322318Z",
     "iopub.status.busy": "2021-11-28T20:13:42.321404Z",
     "iopub.status.idle": "2021-11-28T20:17:14.269466Z",
     "shell.execute_reply": "2021-11-28T20:17:14.268827Z",
     "shell.execute_reply.started": "2021-11-28T05:28:40.369814Z"
    },
    "papermill": {
     "duration": 212.034822,
     "end_time": "2021-11-28T20:17:14.269640",
     "exception": false,
     "start_time": "2021-11-28T20:13:42.234818",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3min 30s, sys: 108 ms, total: 3min 30s\n",
      "Wall time: 3min 31s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(max_depth=19, min_samples_leaf=16, random_state=0)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# best model\n",
    "gradb_model = GradientBoostingClassifier(random_state = 0,\n",
    "                                         max_depth = 19, \n",
    "                                         min_samples_leaf = 16)\n",
    "\n",
    "gradb_model.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "dbdcc031",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T20:17:14.434265Z",
     "iopub.status.busy": "2021-11-28T20:17:14.433418Z",
     "iopub.status.idle": "2021-11-28T20:20:08.845068Z",
     "shell.execute_reply": "2021-11-28T20:20:08.844586Z",
     "shell.execute_reply.started": "2021-11-28T03:42:38.832145Z"
    },
    "papermill": {
     "duration": 174.495452,
     "end_time": "2021-11-28T20:20:08.845187",
     "exception": false,
     "start_time": "2021-11-28T20:17:14.349735",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 53s, sys: 142 ms, total: 2min 53s\n",
      "Wall time: 2min 54s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(max_depth=15, min_samples_leaf=20, random_state=0)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# best model\n",
    "gradb_model = GradientBoostingClassifier(random_state = 0,\n",
    "                                         max_depth = gradb_study.best_trial.params['max_depth'], \n",
    "                                         min_samples_leaf = gradb_study.best_trial.params['min_samples_leaf'])\n",
    "\n",
    "gradb_model.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29221211",
   "metadata": {
    "papermill": {
     "duration": 0.079085,
     "end_time": "2021-11-28T20:20:09.002848",
     "exception": false,
     "start_time": "2021-11-28T20:20:08.923763",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Extreme Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ac079ce6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T20:20:09.168353Z",
     "iopub.status.busy": "2021-11-28T20:20:09.167542Z",
     "iopub.status.idle": "2021-11-28T20:29:26.042986Z",
     "shell.execute_reply": "2021-11-28T20:29:26.042397Z",
     "shell.execute_reply.started": "2021-11-28T05:32:20.978540Z"
    },
    "papermill": {
     "duration": 556.961226,
     "end_time": "2021-11-28T20:29:26.043136",
     "exception": false,
     "start_time": "2021-11-28T20:20:09.081910",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-11-28 20:20:09,165]\u001b[0m A new study created in memory with name: no-name-7eebfcf6-416f-421a-aee6-06c6f184ce0a\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:22:01,194]\u001b[0m Trial 0 finished with value: 0.799074074074074 and parameters: {'max_depth': 16}. Best is trial 0 with value: 0.799074074074074.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:23:49,310]\u001b[0m Trial 1 finished with value: 0.799074074074074 and parameters: {'max_depth': 16}. Best is trial 0 with value: 0.799074074074074.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:25:36,499]\u001b[0m Trial 2 finished with value: 0.799074074074074 and parameters: {'max_depth': 16}. Best is trial 0 with value: 0.799074074074074.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:27:24,433]\u001b[0m Trial 3 finished with value: 0.799074074074074 and parameters: {'max_depth': 16}. Best is trial 0 with value: 0.799074074074074.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:29:25,946]\u001b[0m Trial 4 finished with value: 0.798148148148148 and parameters: {'max_depth': 20}. Best is trial 0 with value: 0.799074074074074.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.799074074074074\n",
      "Best hyperparameters: {'max_depth': 16}\n",
      "CPU times: user 358 ms, sys: 107 ms, total: 465 ms\n",
      "Wall time: 9min 16s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# optuna hyperparameter tuning\n",
    "def objective(trial):\n",
    "    max_depth = trial.suggest_int('max_depth', 10, 20)\n",
    "    xgb_clf = XGBClassifier(random_state = 0, max_depth = max_depth, tree_method = 'gpu_hist')\n",
    "    return sklearn.model_selection.cross_val_score(xgb_clf, X, y, n_jobs = -1, cv = 10).mean()\n",
    "    \n",
    "xgb_study = optuna.create_study(direction='maximize')\n",
    "xgb_study.optimize(objective, n_trials = 5)\n",
    "xgb = xgb_study.best_trial\n",
    "print('Accuracy: {}'.format(xgb.value))\n",
    "print(\"Best hyperparameters: {}\".format(xgb.params))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97c19a4d",
   "metadata": {
    "papermill": {
     "duration": 0.083352,
     "end_time": "2021-11-28T20:29:26.221022",
     "exception": false,
     "start_time": "2021-11-28T20:29:26.137670",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Best Params\n",
    "\n",
    "- Accuracy: 0.7998015873015873\n",
    "- Best hyperparameters: {'max_depth': 14}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "5c0894fd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T20:29:26.392366Z",
     "iopub.status.busy": "2021-11-28T20:29:26.391606Z",
     "iopub.status.idle": "2021-11-28T20:29:33.869175Z",
     "shell.execute_reply": "2021-11-28T20:29:33.869981Z",
     "shell.execute_reply.started": "2021-11-28T05:41:02.298910Z"
    },
    "papermill": {
     "duration": 7.565818,
     "end_time": "2021-11-28T20:29:33.870152",
     "exception": false,
     "start_time": "2021-11-28T20:29:26.304334",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/xgboost/sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[20:29:27] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "CPU times: user 7.59 s, sys: 239 ms, total: 7.83 s\n",
      "Wall time: 7.47 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# best model\n",
    "xgb_model = XGBClassifier(random_state = 0,\n",
    "                           max_depth = 14,\n",
    "                           tree_method = 'gpu_hist')\n",
    "\n",
    "xgb_model.fit(X, y)\n",
    "xgb_model.score(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b705897f",
   "metadata": {
    "papermill": {
     "duration": 0.082614,
     "end_time": "2021-11-28T20:29:34.037122",
     "exception": false,
     "start_time": "2021-11-28T20:29:33.954508",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# AdaBoost "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "1b0ff9d4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T20:29:34.222139Z",
     "iopub.status.busy": "2021-11-28T20:29:34.221318Z",
     "iopub.status.idle": "2021-11-28T20:29:52.285943Z",
     "shell.execute_reply": "2021-11-28T20:29:52.286579Z",
     "shell.execute_reply.started": "2021-11-28T05:41:15.581845Z"
    },
    "papermill": {
     "duration": 18.162851,
     "end_time": "2021-11-28T20:29:52.286764",
     "exception": false,
     "start_time": "2021-11-28T20:29:34.123913",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-11-28 20:29:34,219]\u001b[0m A new study created in memory with name: no-name-4552a77c-b71d-4776-9b05-354ec63a5b6b\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:29:35,389]\u001b[0m Trial 0 finished with value: 0.4352513227513227 and parameters: {'n_estimators': 5}. Best is trial 0 with value: 0.4352513227513227.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:29:38,250]\u001b[0m Trial 1 finished with value: 0.4474206349206349 and parameters: {'n_estimators': 12}. Best is trial 1 with value: 0.4474206349206349.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:29:38,799]\u001b[0m Trial 2 finished with value: 0.38921957671957674 and parameters: {'n_estimators': 2}. Best is trial 1 with value: 0.4474206349206349.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:29:40,145]\u001b[0m Trial 3 finished with value: 0.45436507936507936 and parameters: {'n_estimators': 6}. Best is trial 3 with value: 0.45436507936507936.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:29:42,928]\u001b[0m Trial 4 finished with value: 0.3843915343915344 and parameters: {'n_estimators': 13}. Best is trial 3 with value: 0.45436507936507936.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:29:45,080]\u001b[0m Trial 5 finished with value: 0.4491402116402116 and parameters: {'n_estimators': 10}. Best is trial 3 with value: 0.45436507936507936.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:29:48,276]\u001b[0m Trial 6 finished with value: 0.44484126984126987 and parameters: {'n_estimators': 14}. Best is trial 3 with value: 0.45436507936507936.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:29:48,857]\u001b[0m Trial 7 finished with value: 0.38921957671957674 and parameters: {'n_estimators': 2}. Best is trial 3 with value: 0.45436507936507936.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:29:50,170]\u001b[0m Trial 8 finished with value: 0.45436507936507936 and parameters: {'n_estimators': 6}. Best is trial 3 with value: 0.45436507936507936.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:29:52,277]\u001b[0m Trial 9 finished with value: 0.4491402116402116 and parameters: {'n_estimators': 10}. Best is trial 3 with value: 0.45436507936507936.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.45436507936507936\n",
      "Best hyperparameters: {'n_estimators': 6}\n",
      "CPU times: user 470 ms, sys: 73.1 ms, total: 543 ms\n",
      "Wall time: 18.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# optuna hyperparameter tuning\n",
    "def objective(trial):\n",
    "    n_estimators = trial.suggest_int('n_estimators', 1, 15)\n",
    "    adab_clf = AdaBoostClassifier(random_state = 0, n_estimators = n_estimators)\n",
    "    return sklearn.model_selection.cross_val_score(adab_clf, X, y, n_jobs = -1, cv = 10).mean()\n",
    "    \n",
    "adab_study = optuna.create_study(direction='maximize')\n",
    "adab_study.optimize(objective, n_trials = 10)\n",
    "adab = adab_study.best_trial\n",
    "print('Accuracy: {}'.format(adab.value))\n",
    "print(\"Best hyperparameters: {}\".format(adab.params))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8deb169",
   "metadata": {
    "papermill": {
     "duration": 0.082774,
     "end_time": "2021-11-28T20:29:52.457511",
     "exception": false,
     "start_time": "2021-11-28T20:29:52.374737",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Best Params\n",
    "\n",
    "- Accuracy: 0.45436507936507936\n",
    "- Best hyperparameters: {'n_estimators': 6}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "df50b6c0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T20:29:52.633602Z",
     "iopub.status.busy": "2021-11-28T20:29:52.632857Z",
     "iopub.status.idle": "2021-11-28T20:29:52.795556Z",
     "shell.execute_reply": "2021-11-28T20:29:52.795144Z",
     "shell.execute_reply.started": "2021-11-28T05:41:54.701465Z"
    },
    "papermill": {
     "duration": 0.252608,
     "end_time": "2021-11-28T20:29:52.795669",
     "exception": false,
     "start_time": "2021-11-28T20:29:52.543061",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(n_estimators=6, random_state=0)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# best model\n",
    "adab_model = AdaBoostClassifier(random_state = 0, \n",
    "                                n_estimators = adab_study.best_trial.params['n_estimators'])\n",
    "\n",
    "adab_model.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e15c5f4f",
   "metadata": {
    "papermill": {
     "duration": 0.085214,
     "end_time": "2021-11-28T20:29:52.967436",
     "exception": false,
     "start_time": "2021-11-28T20:29:52.882222",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# MLP Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d99af6f4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T20:29:53.148991Z",
     "iopub.status.busy": "2021-11-28T20:29:53.147981Z",
     "iopub.status.idle": "2021-11-28T20:29:53.150365Z",
     "shell.execute_reply": "2021-11-28T20:29:53.150766Z"
    },
    "papermill": {
     "duration": 0.094797,
     "end_time": "2021-11-28T20:29:53.150962",
     "exception": false,
     "start_time": "2021-11-28T20:29:53.056165",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dda312d6",
   "metadata": {
    "papermill": {
     "duration": 0.086844,
     "end_time": "2021-11-28T20:29:53.325013",
     "exception": false,
     "start_time": "2021-11-28T20:29:53.238169",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "2924dd40",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T20:29:53.504947Z",
     "iopub.status.busy": "2021-11-28T20:29:53.504141Z",
     "iopub.status.idle": "2021-11-28T20:32:49.674223Z",
     "shell.execute_reply": "2021-11-28T20:32:49.673530Z",
     "shell.execute_reply.started": "2021-11-28T05:45:47.901663Z"
    },
    "papermill": {
     "duration": 176.261713,
     "end_time": "2021-11-28T20:32:49.674394",
     "exception": false,
     "start_time": "2021-11-28T20:29:53.412681",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-11-28 20:29:53,501]\u001b[0m A new study created in memory with name: no-name-39dcccb8-5a23-4515-bccf-17e44446ceca\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:30:12,121]\u001b[0m Trial 0 finished with value: 0.7878306878306878 and parameters: {'num_leaves': 20, 'max_depth': 20}. Best is trial 0 with value: 0.7878306878306878.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:30:28,973]\u001b[0m Trial 1 finished with value: 0.7878306878306878 and parameters: {'num_leaves': 18, 'max_depth': 14}. Best is trial 0 with value: 0.7878306878306878.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:30:46,797]\u001b[0m Trial 2 finished with value: 0.7878306878306878 and parameters: {'num_leaves': 10, 'max_depth': 15}. Best is trial 0 with value: 0.7878306878306878.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:31:04,690]\u001b[0m Trial 3 finished with value: 0.7878306878306878 and parameters: {'num_leaves': 12, 'max_depth': 12}. Best is trial 0 with value: 0.7878306878306878.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:31:22,093]\u001b[0m Trial 4 finished with value: 0.7878306878306878 and parameters: {'num_leaves': 13, 'max_depth': 10}. Best is trial 0 with value: 0.7878306878306878.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:31:39,752]\u001b[0m Trial 5 finished with value: 0.7878306878306878 and parameters: {'num_leaves': 19, 'max_depth': 14}. Best is trial 0 with value: 0.7878306878306878.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:31:56,636]\u001b[0m Trial 6 finished with value: 0.7878306878306878 and parameters: {'num_leaves': 15, 'max_depth': 20}. Best is trial 0 with value: 0.7878306878306878.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:32:14,510]\u001b[0m Trial 7 finished with value: 0.7878306878306878 and parameters: {'num_leaves': 13, 'max_depth': 16}. Best is trial 0 with value: 0.7878306878306878.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:32:31,534]\u001b[0m Trial 8 finished with value: 0.7878306878306878 and parameters: {'num_leaves': 11, 'max_depth': 13}. Best is trial 0 with value: 0.7878306878306878.\u001b[0m\n",
      "\u001b[32m[I 2021-11-28 20:32:49,578]\u001b[0m Trial 9 finished with value: 0.7878306878306878 and parameters: {'num_leaves': 10, 'max_depth': 18}. Best is trial 0 with value: 0.7878306878306878.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.7878306878306878\n",
      "Best hyperparameters: {'num_leaves': 20, 'max_depth': 20}\n",
      "CPU times: user 624 ms, sys: 98.5 ms, total: 723 ms\n",
      "Wall time: 2min 56s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# optuna hyperparameter tuning\n",
    "def objective(trial):\n",
    "    num_leaves = trial.suggest_int('num_leaves', 10, 20)\n",
    "    max_depth = trial.suggest_int('max_depth', 10, 20)\n",
    "    lgbm_clf = LGBMClassifier(random_state = 0, )\n",
    "    return sklearn.model_selection.cross_val_score(lgbm_clf, X, y, n_jobs = -1, cv = 10).mean()\n",
    "    \n",
    "lgbm_study = optuna.create_study(direction='maximize')\n",
    "lgbm_study.optimize(objective, n_trials = 10)\n",
    "lgbm = lgbm_study.best_trial\n",
    "print('Accuracy: {}'.format(lgbm.value))\n",
    "print(\"Best hyperparameters: {}\".format(lgbm.params))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0f3ed63",
   "metadata": {
    "papermill": {
     "duration": 0.090221,
     "end_time": "2021-11-28T20:32:49.857580",
     "exception": false,
     "start_time": "2021-11-28T20:32:49.767359",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Best Params\n",
    "\n",
    "- Accuracy: 0.7878306878306878\n",
    "- Best hyperparameters: {'num_leaves': 10, 'max_depth': 16}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "0db63685",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T20:32:50.046737Z",
     "iopub.status.busy": "2021-11-28T20:32:50.045911Z",
     "iopub.status.idle": "2021-11-28T20:32:51.679995Z",
     "shell.execute_reply": "2021-11-28T20:32:51.680741Z",
     "shell.execute_reply.started": "2021-11-28T05:50:33.761221Z"
    },
    "papermill": {
     "duration": 1.732073,
     "end_time": "2021-11-28T20:32:51.680937",
     "exception": false,
     "start_time": "2021-11-28T20:32:49.948864",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.15 s, sys: 62 ms, total: 3.21 s\n",
      "Wall time: 1.63 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(max_depth=20, num_leaves=20, random_state=0)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# best model\n",
    "lgbm_model = LGBMClassifier(random_state = 0,\n",
    "                            max_depth = lgbm_study.best_trial.params['max_depth'],\n",
    "                            num_leaves = lgbm_study.best_trial.params['num_leaves'])\n",
    "\n",
    "lgbm_model.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7a449f0",
   "metadata": {
    "papermill": {
     "duration": 0.096678,
     "end_time": "2021-11-28T20:32:51.871025",
     "exception": false,
     "start_time": "2021-11-28T20:32:51.774347",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Model Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "88aa6cc5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T20:32:52.070313Z",
     "iopub.status.busy": "2021-11-28T20:32:52.069457Z",
     "iopub.status.idle": "2021-11-28T20:36:03.746247Z",
     "shell.execute_reply": "2021-11-28T20:36:03.746868Z",
     "shell.execute_reply.started": "2021-11-28T05:50:47.865687Z"
    },
    "papermill": {
     "duration": 191.780762,
     "end_time": "2021-11-28T20:36:03.747106",
     "exception": false,
     "start_time": "2021-11-28T20:32:51.966344",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/xgboost/sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[20:35:52] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Logistic Regression Accuracy 0.7126322751322751\n",
      "Decision Tree Accuracy 1.0\n",
      "Random Forest Accuracy 1.0\n",
      "Extra Trees Accuracy 1.0\n",
      "Gradient Boosting Accuracy 1.0\n",
      "Extra Gradient Boosting Accuracy 1.0\n",
      "AdaBoost Accuracy 0.426521164021164\n",
      "LightGBM Accuracy 0.9570767195767196\n",
      "Ensemble Accuracy: 1.0\n",
      "CPU times: user 3min 11s, sys: 415 ms, total: 3min 12s\n",
      "Wall time: 3min 11s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "# create ensemble classifier \n",
    "ensemble_model = VotingClassifier(\n",
    "    estimators = [('tree', tree_model), \n",
    "                  ('rf', rf_model), \n",
    "                  ('gradb', gradb_model), \n",
    "                  ('xgb', xgb_model)],\n",
    "    voting = 'hard'\n",
    ")\n",
    "\n",
    "# fit\n",
    "ensemble_model.fit(X, y)\n",
    "\n",
    "# print training accuracy\n",
    "print('Logistic Regression Accuracy', lr_model.score(X, y))\n",
    "print('Decision Tree Accuracy', dt_model.score(X, y))\n",
    "print('Random Forest Accuracy', rf_model.score(X, y))\n",
    "print('Extra Trees Accuracy', tree_model.score(X, y))\n",
    "print('Gradient Boosting Accuracy', gradb_model.score(X, y))\n",
    "print('Extra Gradient Boosting Accuracy', xgb_model.score(X, y))\n",
    "print('AdaBoost Accuracy', adab_model.score(X, y))\n",
    "print('LightGBM Accuracy', lgbm_model.score(X, y))\n",
    "print('Ensemble Accuracy:', ensemble_model.score(X, y))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c4386f8",
   "metadata": {
    "papermill": {
     "duration": 0.091783,
     "end_time": "2021-11-28T20:36:03.935184",
     "exception": false,
     "start_time": "2021-11-28T20:36:03.843401",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- Logistic Regression Accuracy 0.7126322751322751\n",
    "- Decision Tree Accuracy 0.9808201058201058\n",
    "- Random Forest Accuracy 0.9999338624338624\n",
    "- Extra Trees Accuracy 1.0\n",
    "- Gradient Boosting Accuracy 1.0\n",
    "- Extra Gradient Boosting Accuracy 1.0\n",
    "- AdaBoost Accuracy 0.426521164021164\n",
    "- LightGBM Accuracy 0.8923941798941799\n",
    "- Ensemble Accuracy: 1.0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c97dd14",
   "metadata": {
    "papermill": {
     "duration": 0.091556,
     "end_time": "2021-11-28T20:36:04.118978",
     "exception": false,
     "start_time": "2021-11-28T20:36:04.027422",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Save Preprocessor and Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "85a18f07",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T20:36:04.309555Z",
     "iopub.status.busy": "2021-11-28T20:36:04.308845Z",
     "iopub.status.idle": "2021-11-28T20:36:04.311449Z",
     "shell.execute_reply": "2021-11-28T20:36:04.311954Z",
     "shell.execute_reply.started": "2021-11-28T05:55:45.089971Z"
    },
    "papermill": {
     "duration": 0.101032,
     "end_time": "2021-11-28T20:36:04.312100",
     "exception": false,
     "start_time": "2021-11-28T20:36:04.211068",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['forest_cover_scaler2.joblib']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save scaler\n",
    "joblib.dump(scaler, 'forest_cover_scaler2.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "775c1cd8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-11-28T20:36:04.521366Z",
     "iopub.status.busy": "2021-11-28T20:36:04.520428Z",
     "iopub.status.idle": "2021-11-28T20:36:06.939420Z",
     "shell.execute_reply": "2021-11-28T20:36:06.940115Z",
     "shell.execute_reply.started": "2021-11-28T05:55:52.714904Z"
    },
    "papermill": {
     "duration": 2.53173,
     "end_time": "2021-11-28T20:36:06.940304",
     "exception": false,
     "start_time": "2021-11-28T20:36:04.408574",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model written to file.\n"
     ]
    }
   ],
   "source": [
    "joblib.dump(rf_model, 'rf_model_3.joblib')\n",
    "joblib.dump(tree_model, 'tree_model_3.joblib')\n",
    "joblib.dump(gradb_model, 'gradb_model_3.joblib')\n",
    "joblib.dump(xgb_model, 'xgb_model_3.joblib')\n",
    "joblib.dump(adab_model, 'adab_model_3.joblib')\n",
    "joblib.dump(lgbm_model, 'lgbm_model_3.joblib')\n",
    "joblib.dump(ensemble_model, 'ensemble_model_3.joblib')\n",
    "print('Model written to file.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51edda6f",
   "metadata": {
    "papermill": {
     "duration": 0.09229,
     "end_time": "2021-11-28T20:36:07.126515",
     "exception": false,
     "start_time": "2021-11-28T20:36:07.034225",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 8244.884024,
   "end_time": "2021-11-28T20:36:08.743992",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-11-28T18:18:43.859968",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}