Wrapper Method For Feature Selection - Forward and Backward .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "class FeatureSelection:\n",
    "    \n",
    "    def forward_feature_selection(self, X, y, n_features=2):\n",
    "        results = {'best':[]}\n",
    "        feature_list = [] #\n",
    "        while len(feature_list) < n_features:\n",
    "            best = None\n",
    "            for i in range(X.shape[1]):\n",
    "                if i not in feature_list: \n",
    "                    cur_model_result = train_model(X, y, feature_list + [i])\n",
    "                    best = get_best(best, cur_model_result)\n",
    "            feature_list = best['features']\n",
    "            results['best'].append(best)\n",
    "        self.result = compute_best_features(results, n_features)\n",
    "        return results\n",
    "\n",
    "    def backward_feature_selection(self, X, y, n_features=2):\n",
    "        results = {'best':[]}\n",
    "        feature_list = range(X.shape[1])\n",
    "        while len(feature_list) > 1:\n",
    "            best = None\n",
    "            for i in range(len(feature_list)):\n",
    "                idx = feature_list[0:i] + feature_list[i+1:]\n",
    "                best = get_best(best, train_model(X, y, idx))\n",
    "            feature_list = best['features']\n",
    "            results['best'].append(best)\n",
    "        self.result =  compute_best_features(results, n_features)\n",
    "        return results\n",
    "\n",
    "def compute_best_features(results, n_features):\n",
    "    best_result = None\n",
    "    for result in results['best']:\n",
    "        if len(result['features']) <= n_features:\n",
    "            best_result = get_best(best_result, result)\n",
    "    results['best_features'] = best_result['features']\n",
    "    \n",
    "def train_model(X, y, idx):\n",
    "    X_ = X[:, idx]\n",
    "    model = LogisticRegression(random_state=0, solver='lbfgs',multi_class='multinomial').fit(X_, y)\n",
    "    acc = accuracy_score(y, model.predict(X_))\n",
    "    return {'score': acc, 'features': idx}\n",
    "\n",
    "def get_best(b1, b2):\n",
    "    return b2 if b1 == None or b2['score'] > b1['score'] else b1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**********Forward*********\n",
      "{'best_features': [3, 2], 'best': [{'score': 0.96, 'features': [3]}, {'score': 0.9666666666666667, 'features': [3, 2]}]}\n",
      "\n",
      "\n",
      "**********Backward*********\n",
      "{'best_features': [2, 3], 'best': [{'score': 0.9733333333333334, 'features': [1, 2, 3]}, {'score': 0.9666666666666667, 'features': [2, 3]}, {'score': 0.96, 'features': [3]}]}\n"
     ]
    }
   ],
   "source": [
    "from sklearn import datasets\n",
    "iris = datasets.load_iris()\n",
    "X = iris.data \n",
    "Y = iris.target\n",
    "print('**********Forward*********')\n",
    "print(FeatureSelection().forward_feature_selection(X, Y))\n",
    "print('\\n')\n",
    "print('**********Backward*********')\n",
    "print(FeatureSelection().backward_feature_selection(X, Y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}