C_Identify_Feature_Importance_And_Stuff.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:52.835495Z",
     "start_time": "2018-10-02T05:56:52.537295Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder, OrdinalEncoder\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.compose import ColumnTransformer, make_column_transformer\n",
    "from sklearn.pipeline import Pipeline, make_pipeline, FeatureUnion, make_union\n",
    "\n",
    "from sklearn.impute import SimpleImputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:52.862424Z",
     "start_time": "2018-10-02T05:56:52.837490Z"
    }
   },
   "outputs": [],
   "source": [
    "sns.set_style('whitegrid')\n",
    "# sns.set(context=\"paper\", font=\"monospace\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Loading "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:53.349122Z",
     "start_time": "2018-10-02T05:56:52.867410Z"
    }
   },
   "outputs": [],
   "source": [
    "def load_training_dataset():\n",
    "    return pd.read_csv('data/train_LZdllcl.csv', index_col='employee_id')\n",
    "df = load_training_dataset()\n",
    "df = df.rename(columns={'KPIs_met >80%':'is_kpi_met', 'awards_won?':'has_won_awards'})\n",
    "print('Train Data Size :', df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:53.441876Z",
     "start_time": "2018-10-02T05:56:53.353111Z"
    }
   },
   "outputs": [],
   "source": [
    "test = pd.read_csv('data/test_2umaH9m.csv', index_col='employee_id')\n",
    "test = test.rename(columns={'KPIs_met >80%':'is_kpi_met', 'awards_won?':'has_won_awards'})\n",
    "print('Test Data Size :', test.shape)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:53.526649Z",
     "start_time": "2018-10-02T05:56:53.445864Z"
    }
   },
   "outputs": [],
   "source": [
    "test.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-13T19:05:55.603686Z",
     "start_time": "2018-09-13T19:05:55.596707Z"
    }
   },
   "source": [
    "## Data Pre-processing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:53.708507Z",
     "start_time": "2018-10-02T05:56:53.529640Z"
    }
   },
   "outputs": [],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:53.968811Z",
     "start_time": "2018-10-02T05:56:53.712496Z"
    }
   },
   "outputs": [],
   "source": [
    "print(df.info())\n",
    "\n",
    "df['is_promoted'] = df['is_promoted'].astype(np.int8)\n",
    "df['avg_training_score'] = df['avg_training_score'].astype(np.int8)\n",
    "df['has_won_awards'] = df['has_won_awards'].astype(np.int8)\n",
    "df['is_kpi_met'] = df['is_kpi_met'].astype(np.int8)\n",
    "df['length_of_service'] = df['length_of_service'].astype(np.int8)\n",
    "df['age'] = df['age'].astype(np.int8)\n",
    "df['no_of_trainings'] = df['no_of_trainings'].astype(np.int8)\n",
    "\n",
    "df['previous_year_rating'] = df['previous_year_rating'].astype(np.float16)\n",
    "\n",
    "df['department'] = df['department'].astype('category')\n",
    "df['region'] = df['region'].astype('category')\n",
    "df['education'] = df['education'].astype('category')\n",
    "df['gender'] = df['gender'].astype('category')\n",
    "df['recruitment_channel'] = df['recruitment_channel'].astype('category')\n",
    "\n",
    "print('\\nAfter processing:\\n',)\n",
    "print(df.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:54.068545Z",
     "start_time": "2018-10-02T05:56:53.972800Z"
    }
   },
   "outputs": [],
   "source": [
    "print(test.info())\n",
    "\n",
    "# test['is_promoted'] = test['is_promoted'].astype(np.int8)\n",
    "test['avg_training_score'] = test['avg_training_score'].astype(np.int8)\n",
    "test['has_won_awards'] = test['has_won_awards'].astype(np.int8)\n",
    "test['is_kpi_met'] = test['is_kpi_met'].astype(np.int8)\n",
    "test['length_of_service'] = test['length_of_service'].astype(np.int8)\n",
    "test['age'] = test['age'].astype(np.int8)\n",
    "test['no_of_trainings'] = test['no_of_trainings'].astype(np.int8)\n",
    "\n",
    "test['previous_year_rating'] = test['previous_year_rating'].astype(np.float16)\n",
    "\n",
    "test['department'] = test['department'].astype('category')\n",
    "test['region'] = test['region'].astype('category')\n",
    "test['education'] = test['education'].astype('category')\n",
    "test['gender'] = test['gender'].astype('category')\n",
    "test['recruitment_channel'] = test['recruitment_channel'].astype('category')\n",
    "\n",
    "print('\\nAfter processing:\\n',)\n",
    "print(test.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:54.231110Z",
     "start_time": "2018-10-02T05:56:54.071536Z"
    }
   },
   "outputs": [],
   "source": [
    "df_promoted = df[df.is_promoted==1]\n",
    "df_others = df[df.is_promoted==0].sample(df_promoted.shape[0])\n",
    "df_sample = pd.concat([df_promoted, df_others])\n",
    "print('Promoted : ', df_promoted.shape[0], ', Others :', df_others.shape[0], ', Combined Sample :', df_sample.shape[0])\n",
    "df_sample.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:54.519341Z",
     "start_time": "2018-10-02T05:56:54.234102Z"
    }
   },
   "outputs": [],
   "source": [
    "sns.boxplot(x='age', data=df_promoted)\n",
    "df_promoted.age.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:54.706839Z",
     "start_time": "2018-10-02T05:56:54.522333Z"
    }
   },
   "outputs": [],
   "source": [
    "df['is_middle_age'] = df['age'].apply(lambda x: 1 if(x>=29 and x<=38) else 0) #(df['age']>=29 & df['age']<=38)\n",
    "df['is_middle_age'] = df['is_middle_age'].astype(np.int8)\n",
    "df['is_middle_age'].unique()\n",
    "df.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:54.852451Z",
     "start_time": "2018-10-02T05:56:54.710828Z"
    }
   },
   "outputs": [],
   "source": [
    "df.education.fillna(\"Bachelor's\", inplace=True)\n",
    "print(df.education.unique())\n",
    "\n",
    "def numeric_edu(str):\n",
    "    if(str.startswith('Master')): return 2\n",
    "    elif(str.startswith('Bachelor')): return 1\n",
    "    else: return 0\n",
    "\n",
    "df['i_education'] = df.education.apply(numeric_edu)\n",
    "df['i_education'] = df['i_education'].astype(np.int8)\n",
    "print(df['i_education'].unique())\n",
    "df.corr()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rank Features By Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:54.962157Z",
     "start_time": "2018-10-02T05:56:54.855442Z"
    }
   },
   "outputs": [],
   "source": [
    "# Find and Remove attributes with low variance\n",
    "nullList = []\n",
    "for x in df.select_dtypes(include=[np.number]):\n",
    "    p=df[x].std()\n",
    "    if ( p < 0.01 ) :\n",
    "        nullList.append(x)\n",
    "# df = df.drop(nullList, axis=1)\n",
    "nullList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:55.252382Z",
     "start_time": "2018-10-02T05:56:54.966147Z"
    }
   },
   "outputs": [],
   "source": [
    "def numerify(ser):\n",
    "    return LabelEncoder().fit_transform(ser)\n",
    "\n",
    "df2 = df.copy()\n",
    "\n",
    "df2['previous_year_rating'] = df2['previous_year_rating'].fillna(3)\n",
    "\n",
    "# select_categorical_cols = ['department', 'region','education', 'gender', 'recruitment_channel']\n",
    "df2['department'] = numerify(df2['department'])\n",
    "df2['region'] = numerify(df2['region'])\n",
    "df2['education'] = numerify(df2['education'])\n",
    "df2['gender'] = numerify(df2['gender'])\n",
    "df2['recruitment_channel'] = numerify(df2['recruitment_channel'])\n",
    "df2.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:57.038887Z",
     "start_time": "2018-10-02T05:56:55.255373Z"
    }
   },
   "outputs": [],
   "source": [
    "##Split train data-set\n",
    "from sklearn.model_selection import train_test_split\n",
    "x_train, x_test, y_train, y_test = train_test_split(df2[df2.columns.difference(['is_promoted'])], \n",
    "                                                    df2['is_promoted'], \n",
    "                                                    train_size = 0.75, \n",
    "                                                    random_state = 42)\n",
    "\n",
    "## Find Feature Importances\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier(n_estimators=100, n_jobs=-1, random_state=42)\n",
    "rf.fit(x_train, y_train)\n",
    "\n",
    "names = df2.columns.values #select_columns #x_train.columns\n",
    "scores = map(lambda x: round(x, 4), rf.feature_importances_)\n",
    "score_map = zip(scores, names)\n",
    "print(\"Features sorted by their score:\")\n",
    "# for a,b in sorted(score_map, reverse=True):\n",
    "for a,b in sorted(score_map, reverse=True):\n",
    "    print(a,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:58.302241Z",
     "start_time": "2018-10-02T05:56:57.050860Z"
    }
   },
   "outputs": [],
   "source": [
    "features = df2.columns\n",
    "importances = rf.feature_importances_\n",
    "indices = np.argsort(importances) \n",
    "plt.figure(1)\n",
    "plt.title('Feature Importances', fontsize=50)\n",
    "plt.barh(range(len(indices)), importances[indices], color='b', align='center')\n",
    "plt.yticks(range(len(indices)), features[indices], fontsize=50)\n",
    "plt.xlabel('Relative Importance', fontsize=50)\n",
    "fig = plt.gcf()\n",
    "fig.set_size_inches(60, 30, forward=True)\n",
    "# ax = fig.axes[0]\n",
    "# plt.tight_layout()\n",
    "# plt.draw()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:59.313540Z",
     "start_time": "2018-10-02T05:56:58.307226Z"
    }
   },
   "outputs": [],
   "source": [
    "select_numeric_cols = ['avg_training_score', 'has_won_awards', 'is_kpi_met', 'previous_year_rating']\n",
    "select_categorical_cols = ['department', 'region','education', 'gender', 'recruitment_channel'] # select_categorical_cols = []\n",
    "select_columns = select_numeric_cols + select_categorical_cols\n",
    "\n",
    "# Ref.: https://jorisvandenbossche.github.io/blog/2018/05/28/scikit-learn-columntransformer/\n",
    "# Warning: Don't repeat columns to avoid column duplication in resultant vector\n",
    "preprocess = make_column_transformer(\n",
    "    (['previous_year_rating'],make_pipeline(SimpleImputer(strategy='constant',fill_value=3),RobustScaler())),\n",
    "    (list(set(select_numeric_cols) - set(['previous_year_rating'])), RobustScaler()),\n",
    "#     (['education'],make_pipeline(SimpleImputer(strategy='constant',fill_value=\"Bachelor's\"),OrdinalEncoder())),\n",
    "    (['education'],make_pipeline(SimpleImputer(strategy='most_frequent'),OrdinalEncoder())),\n",
    "    (list(set(select_categorical_cols) - set(['education'])), OrdinalEncoder()),    \n",
    ")\n",
    "\n",
    "train_vec = preprocess.fit_transform(df[select_columns])\n",
    "print(train_vec.shape)\n",
    "print(train_vec[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:56:59.342462Z",
     "start_time": "2018-10-02T05:56:59.316531Z"
    }
   },
   "outputs": [],
   "source": [
    "# Split train data-set\n",
    "from sklearn.model_selection import train_test_split\n",
    "x_train, x_test, y_train, y_test = train_test_split(train_vec, \n",
    "                                                    df['is_promoted'], \n",
    "                                                    train_size = 0.75, \n",
    "                                                    random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:57:01.162812Z",
     "start_time": "2018-10-02T05:56:59.349441Z"
    }
   },
   "outputs": [],
   "source": [
    "## Find Feature Importances\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier(n_estimators=100, n_jobs=-1, random_state=42)\n",
    "rf.fit(x_train, y_train)\n",
    "\n",
    "names = select_columns #x_train.columns\n",
    "scores = map(lambda x: round(x, 4), rf.feature_importances_)\n",
    "score_map = zip(scores, names)\n",
    "print(\"Features sorted by their score:\")\n",
    "# for a,b in sorted(score_map, reverse=True):\n",
    "for a,b in sorted(score_map, reverse=True):\n",
    "    print(a,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-18T18:10:21.560147Z",
     "start_time": "2018-09-18T18:10:21.545185Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:57:01.227635Z",
     "start_time": "2018-10-02T05:57:01.165798Z"
    }
   },
   "outputs": [],
   "source": [
    "test_vec = preprocess.transform(test[select_columns])\n",
    "test_vec[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LGBoost for Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:57:01.857140Z",
     "start_time": "2018-10-02T05:57:01.232619Z"
    }
   },
   "outputs": [],
   "source": [
    "from lightgbm import LGBMClassifier\n",
    "\n",
    "from sklearn.model_selection import cross_val_score, cross_validate\n",
    "\n",
    "lgb_clf=LGBMClassifier(random_state=42, silent=False, class_weight={0:1, 1:5}, learning_rate=.05, n_estimators=45)\n",
    "\n",
    "scores = cross_val_score(lgb_clf, x_train, y_train, cv=3, scoring='f1')\n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:57:02.138390Z",
     "start_time": "2018-10-02T05:57:01.861131Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "\n",
    "lgb_clf.fit(x_train, y_train)\n",
    "preds = lgb_clf.predict(x_test)\n",
    "\n",
    "print('Accuracy = ', np.mean(preds == y_test))\n",
    "\n",
    "creport = classification_report(y_test, preds, target_names=['Not Promoted', 'Promoted'])\n",
    "print(creport)\n",
    "\n",
    "confusion_matrix(y_test, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:57:02.150358Z",
     "start_time": "2018-10-02T05:57:02.141383Z"
    }
   },
   "outputs": [],
   "source": [
    "def conv2df(preds):\n",
    "    df = pd.DataFrame(data={\n",
    "        'employee_id': test.index.values,\n",
    "        'is_promoted': preds\n",
    "    })\n",
    "    df['is_promoted'] = df['is_promoted'].astype(np.int8)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:57:02.496010Z",
     "start_time": "2018-10-02T05:57:02.153350Z"
    }
   },
   "outputs": [],
   "source": [
    "conv2df(lgb_clf.predict(test_vec)).to_csv('data/output/final_submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-01T04:52:11.970633Z",
     "start_time": "2018-10-01T04:52:11.802076Z"
    }
   },
   "source": [
    "## Model Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:57:02.505963Z",
     "start_time": "2018-10-02T05:57:02.498967Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "# from sklearn.svm import SVC\n",
    "# from sklearn.naive_bayes import GaussianNB\n",
    "# from sklearn.ensemble import BaggingClassifier\n",
    "# from sklearn.ensemble import VotingClassifier\n",
    "\n",
    "LOGISTIC_REGRESSION = 'LogisticRegression'\n",
    "DECISION_TREE = 'DecisionTree'\n",
    "KNN = 'KNN'\n",
    "RANDOM_FOREST = 'RandomForest'\n",
    "GRADIENT_BOOSTING = 'GradientBoosting'\n",
    "ADA_BOOST = 'AdaBoost'\n",
    "MLP = 'NeuralNetwork'\n",
    "LGBM = 'LGBM'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:57:41.915342Z",
     "start_time": "2018-10-02T05:57:02.508940Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "learners = [\n",
    "    LogisticRegression(random_state=42), \n",
    "    DecisionTreeClassifier(random_state=42), \n",
    "    KNeighborsClassifier(),\n",
    "    RandomForestClassifier(random_state=42), \n",
    "    GradientBoostingClassifier(random_state=42), \n",
    "    AdaBoostClassifier(random_state=42), \n",
    "    MLPClassifier(random_state=42),\n",
    "    LGBMClassifier(random_state=42)\n",
    "]\n",
    "\n",
    "labels = [\n",
    "    LOGISTIC_REGRESSION, \n",
    "    DECISION_TREE, \n",
    "    KNN, \n",
    "    RANDOM_FOREST, \n",
    "    GRADIENT_BOOSTING, \n",
    "    ADA_BOOST, \n",
    "    MLP,\n",
    "    LGBM\n",
    "]\n",
    "\n",
    "lFrame = pd.DataFrame(columns=['Learner', 'scoreAUC'])\n",
    "i=0\n",
    "for learner, label in zip(learners, labels):\n",
    "    learner.fit(x_train, y_train)\n",
    "    y_preds = learner.predict_proba(x_test)[:, 1]\n",
    "    score = roc_auc_score(y_test, y_preds)\n",
    "    lFrame.loc[i] = [label, score]\n",
    "    i += 1\n",
    "    print('Classifier : <',label,'> Score AUC: ', score)\n",
    "    \n",
    "lFrame.sort_values('scoreAUC', ascending=False)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-01T08:27:24.642143Z",
     "start_time": "2018-10-01T08:27:24.638154Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T05:57:41.975196Z",
     "start_time": "2018-10-02T05:57:41.919331Z"
    }
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "    ADA_BOOST:  { 'n_estimators': range(10, 100, 10),\n",
    "                   'learning_rate': [0.01, 0.05, 0.1, 0.5, 1.0] },\n",
    "    \n",
    "    GRADIENT_BOOSTING: { 'n_estimators': [25, 50, 100], \n",
    "                          'learning_rate': [0.001, 0.01, 0.02, 0.03, 0.04, 0.05, 0.1], \n",
    "                          'max_depth': [1, 2, 3, 5, 7, 9, 10],},\n",
    "    \n",
    "    RANDOM_FOREST:  {\"max_depth\": [3, 5, 7, 9, 11], \n",
    "                      \"min_samples_leaf\": [1, 3, 5], \n",
    "                      \"criterion\": [\"gini\", \"entropy\"],  \n",
    "                      \"n_estimators\" : [25, 50, 100] },\n",
    "    \n",
    "    MLP : {'activation' : ['identity', 'logistic', 'tanh', 'relu'],\n",
    "           'solver' : ['lbfgs', 'sgd', 'adam'],\n",
    "           'learning_rate' : ['constant', 'invscaling', 'adaptive'],\n",
    "           'early_stopping' : [True, False] },\n",
    "    \n",
    "    KNN : {'n_neighbors': [2,3,4,5]},\n",
    "    \n",
    "    LOGISTIC_REGRESSION : {'C': range(1, 10, 1)},\n",
    "    \n",
    "    DECISION_TREE : { \"criterion\": [\"gini\", \"entropy\"],\n",
    "                       \"min_samples_split\": [2, 3, 5],\n",
    "                       \"max_depth\": [2, 3, 5, 7, 9],\n",
    "                       \"min_samples_leaf\": [1, 3, 5],\n",
    "                       \"max_leaf_nodes\": [3, 5, 7,9, 11], },\n",
    "    \n",
    "    LGBM : { \"class_weight\" : [{0:1, 1:1}, {0:1, 1:5}],\n",
    "             \"learning_rate\" : [0.025, 0.05, 0.1, 0.25, 0.5],\n",
    "             \"n_estimators\" : [25, 45, 75, 100] }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T06:35:09.450006Z",
     "start_time": "2018-10-02T05:57:41.979171Z"
    }
   },
   "outputs": [],
   "source": [
    "# Ref.: https://github.com/Hichambd/Kaggle-Santander-Customer-Satisfaction/blob/master/Santander%20Customer%20Satisfaction.ipynb\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "classifiers = {\n",
    "    LOGISTIC_REGRESSION : LogisticRegression(random_state=42), \n",
    "#     DECISION_TREE : DecisionTreeClassifier(random_state=42), \n",
    "    KNN : KNeighborsClassifier(), \n",
    "#     RANDOM_FOREST : RandomForestClassifier(random_state=42), \n",
    "#     GRADIENT_BOOSTING : GradientBoostingClassifier(random_state=42),  \n",
    "    ADA_BOOST : AdaBoostClassifier(random_state=42), \n",
    "    MLP : MLPClassifier(random_state=42),\n",
    "    LGBM : LGBMClassifier(random_state=42)\n",
    "}\n",
    "\n",
    "clf_list = []\n",
    "i=0\n",
    "max=len(classifiers)\n",
    "for name,clf in classifiers.items():\n",
    "    print(\"name={}, clf={}, params={}\".format(name,clf,params.get(name)))\n",
    "    gscv = GridSearchCV(clf, param_grid=params.get(name))\n",
    "    gscv.fit(x_train, y_train)\n",
    "    classifiers[name] = gscv.best_estimator_\n",
    "    y_preds = gscv.predict(x_test)\n",
    "    f1score = f1_score(y_test, y_preds)\n",
    "    print(\"{}'s classifier's Best f1score is \".format(name), f1score)    \n",
    "    dict1 = {\n",
    "        'Classifier': name,\n",
    "        'F1Score': f1score,\n",
    "        'BestScore': gscv.best_score_,\n",
    "        'BestParams': gscv.best_params_,\n",
    "        'BestEstimator': gscv.best_estimator_\n",
    "    }\n",
    "    clf_list.append(dict1)\n",
    "    i = i+1\n",
    "    print(\"Completed {}/{}.\".format(i,max))\n",
    "\n",
    "clfDataFrame = pd.DataFrame(clf_list)\n",
    "# clfDataFrame.to_csv('sortedLearners.csv', sep=',', encoding='utf-8')\n",
    "clfDataFrame.sort_values('F1Score', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-02T06:35:09.460976Z",
     "start_time": "2018-10-02T06:35:09.452998Z"
    }
   },
   "outputs": [],
   "source": [
    "for k,v in classifiers.items():\n",
    "    print(k,v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}