data-preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download the dataset from kaggle and unzip it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !kaggle datasets download -d bhavesh907/crop-classificationcs2292017usgscroplanddata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !mv ./crop-classificationcs2292017usgscroplanddata.zip ./Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !7z x ./Dataset/crop-classificationcs2292017usgscroplanddata.zip "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from osgeo import gdal, gdal_array\n",
    "import pandas as pd\n",
    "import operator\n",
    "import gc\n",
    "import os\n",
    "\n",
    "roi_ds = gdal.Open('./Dataset/cdl2017.tiff', gdal.GA_ReadOnly)\n",
    "\n",
    "#train_ds = gdal.Open('D:/neurafarms/downloaded_sat_images/rose_mlready/0306.tiff', gdal.GA_ReadOnly)\n",
    "\n",
    "roi = roi_ds.GetRasterBand(1).ReadAsArray()\n",
    "\n",
    "# How many pixels are in each class?\n",
    "classes = np.unique(roi)\n",
    "\n",
    "\n",
    "# Iterate over all class labels in the ROI image, printing out some information\n",
    "# for c in classes:\n",
    "#     print('Class {c} contains {n} pixels'.format(c=c,\n",
    "#                                                  n=(roi == c).sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 6 classes and pixel counts \n",
      " [(225, 617040), (121, 766044), (36, 1543068), (75, 4729104), (69, 8311968), (255, 9381144)]\n"
     ]
    }
   ],
   "source": [
    "dict = {}\n",
    "for c in classes:\n",
    "    dict[c] = (roi == c).sum()\n",
    "sorted_x = sorted(dict.items(), key=operator.itemgetter(1))\n",
    "print(\"Top 6 classes and pixel counts \\n\",sorted_x[-6:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Select top  5 classes exclude 255 class label\n",
    "\n",
    "top_classes = [69,75,36,121,225]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We have 28406694 samples\n",
      "The training data include 55 classes: [  1   2   4  21  23  24  28  33  36  37  42  48  49  54  57  59  61  66\n",
      "  67  69  71  72  75  76  77 111 121 122 123 124 131 152 176 190 195 204\n",
      " 205 206 208 209 212 213 216 217 218 220 224 225 226 227 236 237 238 242\n",
      " 255]\n"
     ]
    }
   ],
   "source": [
    "# Find how many non-zero entries we have -- i.e. how many training data samples?\n",
    "n_samples = (roi > 0).sum()\n",
    "print('We have {n} samples'.format(n=n_samples))\n",
    "\n",
    "# What are our classification labels?\n",
    "labels = np.unique(roi[roi > 0])\n",
    "print('The training data include {n} classes: {classes}'.format(n=labels.size, \n",
    "                                                                classes=labels))\n",
    "# We will need a \"X\" matrix containing our features, and a \"y\" array containing our labels\n",
    "#     These will have n_samples rows\n",
    "#     In other languages we would need to allocate these and them loop to fill them, but NumPy can be faster\n",
    "\n",
    "#X = img_b1[roi > 0, :]  \n",
    "y = roi[roi > 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "images = ['./Dataset/20170306.tiff',\n",
    "          './Dataset/20170410.tiff',\n",
    "          './Dataset/20170601.tiff',\n",
    "          './Dataset/20170615.tiff',\n",
    "          './Dataset/20170708.tiff',\n",
    "          './Dataset/20170807.tiff',\n",
    "          './Dataset/20170905.tiff',\n",
    "          './Dataset/20170923.tiff',\n",
    "          './Dataset/20171015.tiff',\n",
    "          './Dataset/20171207.tiff']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading class 69\n",
      "69\n",
      "./Dataset/20170306.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170410.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170601.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170615.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170708.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170807.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170905.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170923.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20171015.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20171207.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "75\n",
      "./Dataset/20170306.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170410.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170601.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170615.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170708.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170807.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170905.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170923.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20171015.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20171207.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "36\n",
      "./Dataset/20170306.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170410.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170601.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170615.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170708.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170807.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170905.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170923.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20171015.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20171207.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "121\n",
      "./Dataset/20170306.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170410.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170601.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170615.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170708.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170807.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170905.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170923.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20171015.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20171207.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "225\n",
      "./Dataset/20170306.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170410.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170601.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170615.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170708.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170807.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170905.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20170923.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20171015.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n",
      "./Dataset/20171207.tiff\n",
      "5718 4969\n",
      "(4969, 5718, 5)\n"
     ]
    }
   ],
   "source": [
    "#69,75,36,121,225\n",
    "print(\"Reading class 69\")\n",
    "final = pd.DataFrame()\n",
    "\n",
    "for c in top_classes:\n",
    "    \n",
    "    temp = pd.DataFrame()\n",
    "    \n",
    "    print(c)\n",
    "    \n",
    "    for img in images:\n",
    "\n",
    "        print(img)\n",
    "\n",
    "        train_ds = gdal.Open(img, gdal.GA_ReadOnly)\n",
    "\n",
    "        print(train_ds.RasterXSize,train_ds.RasterYSize)\n",
    "\n",
    "        img_b1 = np.zeros((train_ds.RasterYSize, train_ds.RasterXSize, train_ds.RasterCount),\n",
    "                       gdal_array.GDALTypeCodeToNumericTypeCode(train_ds.GetRasterBand(1).DataType))\n",
    "        \n",
    "        for b in range(img_b1.shape[2]):\n",
    "            img_b1[:, :, b] = train_ds.GetRasterBand(b + 1).ReadAsArray()\n",
    "        \n",
    "        print(img_b1.shape)\n",
    "\n",
    "\n",
    "        Xt = img_b1[roi==c, :] \n",
    "        \n",
    "        Xt1 = pd.DataFrame(Xt)\n",
    "        \n",
    "        Xt2 = Xt1.sample(n=100000)\n",
    "        \n",
    "        Xt2.reset_index(drop=True,inplace=True)\n",
    "        \n",
    "        temp = pd.concat([Xt2,temp],axis=1)\n",
    "        \n",
    "        temp[\"class\"] = c\n",
    "        #temp.reset_index(drop=True,inplace=True)\n",
    "      \n",
    "    final = pd.concat([temp,final],axis=0)\n",
    "    final.reset_index(drop=True,inplace=True)\n",
    "    \n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>...</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4077.0</td>\n",
       "      <td>3801.0</td>\n",
       "      <td>3588.0</td>\n",
       "      <td>3341.0</td>\n",
       "      <td>3509.0</td>\n",
       "      <td>9578.0</td>\n",
       "      <td>10052.0</td>\n",
       "      <td>10280.0</td>\n",
       "      <td>8968.0</td>\n",
       "      <td>8832.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4815.0</td>\n",
       "      <td>2918.0</td>\n",
       "      <td>4605.0</td>\n",
       "      <td>12363.0</td>\n",
       "      <td>5228.0</td>\n",
       "      <td>4855.0</td>\n",
       "      <td>5013.0</td>\n",
       "      <td>5556.0</td>\n",
       "      <td>6514.0</td>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3400.0</td>\n",
       "      <td>3065.0</td>\n",
       "      <td>1710.0</td>\n",
       "      <td>3169.0</td>\n",
       "      <td>7538.0</td>\n",
       "      <td>7435.0</td>\n",
       "      <td>7382.0</td>\n",
       "      <td>7149.0</td>\n",
       "      <td>6712.0</td>\n",
       "      <td>6667.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4802.0</td>\n",
       "      <td>2935.0</td>\n",
       "      <td>4860.0</td>\n",
       "      <td>11249.0</td>\n",
       "      <td>4058.0</td>\n",
       "      <td>3586.0</td>\n",
       "      <td>1835.0</td>\n",
       "      <td>4208.0</td>\n",
       "      <td>9019.0</td>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4518.0</td>\n",
       "      <td>3882.0</td>\n",
       "      <td>3839.0</td>\n",
       "      <td>3623.0</td>\n",
       "      <td>3788.0</td>\n",
       "      <td>6678.0</td>\n",
       "      <td>5938.0</td>\n",
       "      <td>6144.0</td>\n",
       "      <td>5540.0</td>\n",
       "      <td>5463.0</td>\n",
       "      <td>...</td>\n",
       "      <td>9226.0</td>\n",
       "      <td>8795.0</td>\n",
       "      <td>8145.0</td>\n",
       "      <td>8119.0</td>\n",
       "      <td>4128.0</td>\n",
       "      <td>3423.0</td>\n",
       "      <td>1559.0</td>\n",
       "      <td>3836.0</td>\n",
       "      <td>12437.0</td>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4078.0</td>\n",
       "      <td>3398.0</td>\n",
       "      <td>3342.0</td>\n",
       "      <td>2988.0</td>\n",
       "      <td>3025.0</td>\n",
       "      <td>5294.0</td>\n",
       "      <td>4518.0</td>\n",
       "      <td>3827.0</td>\n",
       "      <td>4085.0</td>\n",
       "      <td>4035.0</td>\n",
       "      <td>...</td>\n",
       "      <td>5541.0</td>\n",
       "      <td>3926.0</td>\n",
       "      <td>5093.0</td>\n",
       "      <td>8735.0</td>\n",
       "      <td>4712.0</td>\n",
       "      <td>4614.0</td>\n",
       "      <td>2746.0</td>\n",
       "      <td>4830.0</td>\n",
       "      <td>8588.0</td>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4403.0</td>\n",
       "      <td>3869.0</td>\n",
       "      <td>3487.0</td>\n",
       "      <td>3553.0</td>\n",
       "      <td>4081.0</td>\n",
       "      <td>6378.0</td>\n",
       "      <td>5924.0</td>\n",
       "      <td>6098.0</td>\n",
       "      <td>5600.0</td>\n",
       "      <td>5386.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8122.0</td>\n",
       "      <td>7147.0</td>\n",
       "      <td>6846.0</td>\n",
       "      <td>7443.0</td>\n",
       "      <td>4279.0</td>\n",
       "      <td>3817.0</td>\n",
       "      <td>2050.0</td>\n",
       "      <td>4205.0</td>\n",
       "      <td>12931.0</td>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499995</th>\n",
       "      <td>4266.0</td>\n",
       "      <td>3841.0</td>\n",
       "      <td>3807.0</td>\n",
       "      <td>3492.0</td>\n",
       "      <td>3795.0</td>\n",
       "      <td>5265.0</td>\n",
       "      <td>4319.0</td>\n",
       "      <td>3497.0</td>\n",
       "      <td>3859.0</td>\n",
       "      <td>5019.0</td>\n",
       "      <td>...</td>\n",
       "      <td>5763.0</td>\n",
       "      <td>4397.0</td>\n",
       "      <td>5285.0</td>\n",
       "      <td>7629.0</td>\n",
       "      <td>6406.0</td>\n",
       "      <td>6160.0</td>\n",
       "      <td>6222.0</td>\n",
       "      <td>5642.0</td>\n",
       "      <td>5474.0</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499996</th>\n",
       "      <td>3974.0</td>\n",
       "      <td>3717.0</td>\n",
       "      <td>2986.0</td>\n",
       "      <td>3403.0</td>\n",
       "      <td>4611.0</td>\n",
       "      <td>7109.0</td>\n",
       "      <td>6415.0</td>\n",
       "      <td>6497.0</td>\n",
       "      <td>5964.0</td>\n",
       "      <td>6346.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7474.0</td>\n",
       "      <td>6813.0</td>\n",
       "      <td>6977.0</td>\n",
       "      <td>7597.0</td>\n",
       "      <td>5239.0</td>\n",
       "      <td>4760.0</td>\n",
       "      <td>4163.0</td>\n",
       "      <td>4797.0</td>\n",
       "      <td>5710.0</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499997</th>\n",
       "      <td>4200.0</td>\n",
       "      <td>3589.0</td>\n",
       "      <td>3595.0</td>\n",
       "      <td>3508.0</td>\n",
       "      <td>3776.0</td>\n",
       "      <td>7275.0</td>\n",
       "      <td>7232.0</td>\n",
       "      <td>7675.0</td>\n",
       "      <td>7041.0</td>\n",
       "      <td>7157.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8149.0</td>\n",
       "      <td>7354.0</td>\n",
       "      <td>7577.0</td>\n",
       "      <td>8455.0</td>\n",
       "      <td>6876.0</td>\n",
       "      <td>6585.0</td>\n",
       "      <td>6435.0</td>\n",
       "      <td>5889.0</td>\n",
       "      <td>5727.0</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499998</th>\n",
       "      <td>4183.0</td>\n",
       "      <td>3463.0</td>\n",
       "      <td>3674.0</td>\n",
       "      <td>3451.0</td>\n",
       "      <td>3615.0</td>\n",
       "      <td>6829.0</td>\n",
       "      <td>6644.0</td>\n",
       "      <td>6918.0</td>\n",
       "      <td>6959.0</td>\n",
       "      <td>7425.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8731.0</td>\n",
       "      <td>7728.0</td>\n",
       "      <td>7720.0</td>\n",
       "      <td>8194.0</td>\n",
       "      <td>5954.0</td>\n",
       "      <td>5531.0</td>\n",
       "      <td>4750.0</td>\n",
       "      <td>5113.0</td>\n",
       "      <td>5786.0</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499999</th>\n",
       "      <td>4607.0</td>\n",
       "      <td>4225.0</td>\n",
       "      <td>4081.0</td>\n",
       "      <td>3686.0</td>\n",
       "      <td>3685.0</td>\n",
       "      <td>4754.0</td>\n",
       "      <td>3848.0</td>\n",
       "      <td>2830.0</td>\n",
       "      <td>3681.0</td>\n",
       "      <td>6408.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8845.0</td>\n",
       "      <td>8291.0</td>\n",
       "      <td>8386.0</td>\n",
       "      <td>8443.0</td>\n",
       "      <td>7280.0</td>\n",
       "      <td>7270.0</td>\n",
       "      <td>7357.0</td>\n",
       "      <td>6570.0</td>\n",
       "      <td>6306.0</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>500000 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             0       1       2       3       4       0        1        2  \\\n",
       "0       4077.0  3801.0  3588.0  3341.0  3509.0  9578.0  10052.0  10280.0   \n",
       "1       3400.0  3065.0  1710.0  3169.0  7538.0  7435.0   7382.0   7149.0   \n",
       "2       4518.0  3882.0  3839.0  3623.0  3788.0  6678.0   5938.0   6144.0   \n",
       "3       4078.0  3398.0  3342.0  2988.0  3025.0  5294.0   4518.0   3827.0   \n",
       "4       4403.0  3869.0  3487.0  3553.0  4081.0  6378.0   5924.0   6098.0   \n",
       "...        ...     ...     ...     ...     ...     ...      ...      ...   \n",
       "499995  4266.0  3841.0  3807.0  3492.0  3795.0  5265.0   4319.0   3497.0   \n",
       "499996  3974.0  3717.0  2986.0  3403.0  4611.0  7109.0   6415.0   6497.0   \n",
       "499997  4200.0  3589.0  3595.0  3508.0  3776.0  7275.0   7232.0   7675.0   \n",
       "499998  4183.0  3463.0  3674.0  3451.0  3615.0  6829.0   6644.0   6918.0   \n",
       "499999  4607.0  4225.0  4081.0  3686.0  3685.0  4754.0   3848.0   2830.0   \n",
       "\n",
       "             3       4  ...       1       2       3        4       0       1  \\\n",
       "0       8968.0  8832.0  ...  4815.0  2918.0  4605.0  12363.0  5228.0  4855.0   \n",
       "1       6712.0  6667.0  ...  4802.0  2935.0  4860.0  11249.0  4058.0  3586.0   \n",
       "2       5540.0  5463.0  ...  9226.0  8795.0  8145.0   8119.0  4128.0  3423.0   \n",
       "3       4085.0  4035.0  ...  5541.0  3926.0  5093.0   8735.0  4712.0  4614.0   \n",
       "4       5600.0  5386.0  ...  8122.0  7147.0  6846.0   7443.0  4279.0  3817.0   \n",
       "...        ...     ...  ...     ...     ...     ...      ...     ...     ...   \n",
       "499995  3859.0  5019.0  ...  5763.0  4397.0  5285.0   7629.0  6406.0  6160.0   \n",
       "499996  5964.0  6346.0  ...  7474.0  6813.0  6977.0   7597.0  5239.0  4760.0   \n",
       "499997  7041.0  7157.0  ...  8149.0  7354.0  7577.0   8455.0  6876.0  6585.0   \n",
       "499998  6959.0  7425.0  ...  8731.0  7728.0  7720.0   8194.0  5954.0  5531.0   \n",
       "499999  3681.0  6408.0  ...  8845.0  8291.0  8386.0   8443.0  7280.0  7270.0   \n",
       "\n",
       "             2       3        4  class  \n",
       "0       5013.0  5556.0   6514.0    225  \n",
       "1       1835.0  4208.0   9019.0    225  \n",
       "2       1559.0  3836.0  12437.0    225  \n",
       "3       2746.0  4830.0   8588.0    225  \n",
       "4       2050.0  4205.0  12931.0    225  \n",
       "...        ...     ...      ...    ...  \n",
       "499995  6222.0  5642.0   5474.0     69  \n",
       "499996  4163.0  4797.0   5710.0     69  \n",
       "499997  6435.0  5889.0   5727.0     69  \n",
       "499998  4750.0  5113.0   5786.0     69  \n",
       "499999  7357.0  6570.0   6306.0     69  \n",
       "\n",
       "[500000 rows x 51 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "final.columns = ['col_'+str(i) for i in range(51)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_0</th>\n",
       "      <th>col_1</th>\n",
       "      <th>col_2</th>\n",
       "      <th>col_3</th>\n",
       "      <th>col_4</th>\n",
       "      <th>col_5</th>\n",
       "      <th>col_6</th>\n",
       "      <th>col_7</th>\n",
       "      <th>col_8</th>\n",
       "      <th>col_9</th>\n",
       "      <th>...</th>\n",
       "      <th>col_41</th>\n",
       "      <th>col_42</th>\n",
       "      <th>col_43</th>\n",
       "      <th>col_44</th>\n",
       "      <th>col_45</th>\n",
       "      <th>col_46</th>\n",
       "      <th>col_47</th>\n",
       "      <th>col_48</th>\n",
       "      <th>col_49</th>\n",
       "      <th>col_50</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4077.0</td>\n",
       "      <td>3801.0</td>\n",
       "      <td>3588.0</td>\n",
       "      <td>3341.0</td>\n",
       "      <td>3509.0</td>\n",
       "      <td>9578.0</td>\n",
       "      <td>10052.0</td>\n",
       "      <td>10280.0</td>\n",
       "      <td>8968.0</td>\n",
       "      <td>8832.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4815.0</td>\n",
       "      <td>2918.0</td>\n",
       "      <td>4605.0</td>\n",
       "      <td>12363.0</td>\n",
       "      <td>5228.0</td>\n",
       "      <td>4855.0</td>\n",
       "      <td>5013.0</td>\n",
       "      <td>5556.0</td>\n",
       "      <td>6514.0</td>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3400.0</td>\n",
       "      <td>3065.0</td>\n",
       "      <td>1710.0</td>\n",
       "      <td>3169.0</td>\n",
       "      <td>7538.0</td>\n",
       "      <td>7435.0</td>\n",
       "      <td>7382.0</td>\n",
       "      <td>7149.0</td>\n",
       "      <td>6712.0</td>\n",
       "      <td>6667.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4802.0</td>\n",
       "      <td>2935.0</td>\n",
       "      <td>4860.0</td>\n",
       "      <td>11249.0</td>\n",
       "      <td>4058.0</td>\n",
       "      <td>3586.0</td>\n",
       "      <td>1835.0</td>\n",
       "      <td>4208.0</td>\n",
       "      <td>9019.0</td>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4518.0</td>\n",
       "      <td>3882.0</td>\n",
       "      <td>3839.0</td>\n",
       "      <td>3623.0</td>\n",
       "      <td>3788.0</td>\n",
       "      <td>6678.0</td>\n",
       "      <td>5938.0</td>\n",
       "      <td>6144.0</td>\n",
       "      <td>5540.0</td>\n",
       "      <td>5463.0</td>\n",
       "      <td>...</td>\n",
       "      <td>9226.0</td>\n",
       "      <td>8795.0</td>\n",
       "      <td>8145.0</td>\n",
       "      <td>8119.0</td>\n",
       "      <td>4128.0</td>\n",
       "      <td>3423.0</td>\n",
       "      <td>1559.0</td>\n",
       "      <td>3836.0</td>\n",
       "      <td>12437.0</td>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4078.0</td>\n",
       "      <td>3398.0</td>\n",
       "      <td>3342.0</td>\n",
       "      <td>2988.0</td>\n",
       "      <td>3025.0</td>\n",
       "      <td>5294.0</td>\n",
       "      <td>4518.0</td>\n",
       "      <td>3827.0</td>\n",
       "      <td>4085.0</td>\n",
       "      <td>4035.0</td>\n",
       "      <td>...</td>\n",
       "      <td>5541.0</td>\n",
       "      <td>3926.0</td>\n",
       "      <td>5093.0</td>\n",
       "      <td>8735.0</td>\n",
       "      <td>4712.0</td>\n",
       "      <td>4614.0</td>\n",
       "      <td>2746.0</td>\n",
       "      <td>4830.0</td>\n",
       "      <td>8588.0</td>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4403.0</td>\n",
       "      <td>3869.0</td>\n",
       "      <td>3487.0</td>\n",
       "      <td>3553.0</td>\n",
       "      <td>4081.0</td>\n",
       "      <td>6378.0</td>\n",
       "      <td>5924.0</td>\n",
       "      <td>6098.0</td>\n",
       "      <td>5600.0</td>\n",
       "      <td>5386.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8122.0</td>\n",
       "      <td>7147.0</td>\n",
       "      <td>6846.0</td>\n",
       "      <td>7443.0</td>\n",
       "      <td>4279.0</td>\n",
       "      <td>3817.0</td>\n",
       "      <td>2050.0</td>\n",
       "      <td>4205.0</td>\n",
       "      <td>12931.0</td>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 51 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    col_0   col_1   col_2   col_3   col_4   col_5    col_6    col_7   col_8  \\\n",
       "0  4077.0  3801.0  3588.0  3341.0  3509.0  9578.0  10052.0  10280.0  8968.0   \n",
       "1  3400.0  3065.0  1710.0  3169.0  7538.0  7435.0   7382.0   7149.0  6712.0   \n",
       "2  4518.0  3882.0  3839.0  3623.0  3788.0  6678.0   5938.0   6144.0  5540.0   \n",
       "3  4078.0  3398.0  3342.0  2988.0  3025.0  5294.0   4518.0   3827.0  4085.0   \n",
       "4  4403.0  3869.0  3487.0  3553.0  4081.0  6378.0   5924.0   6098.0  5600.0   \n",
       "\n",
       "    col_9  ...  col_41  col_42  col_43   col_44  col_45  col_46  col_47  \\\n",
       "0  8832.0  ...  4815.0  2918.0  4605.0  12363.0  5228.0  4855.0  5013.0   \n",
       "1  6667.0  ...  4802.0  2935.0  4860.0  11249.0  4058.0  3586.0  1835.0   \n",
       "2  5463.0  ...  9226.0  8795.0  8145.0   8119.0  4128.0  3423.0  1559.0   \n",
       "3  4035.0  ...  5541.0  3926.0  5093.0   8735.0  4712.0  4614.0  2746.0   \n",
       "4  5386.0  ...  8122.0  7147.0  6846.0   7443.0  4279.0  3817.0  2050.0   \n",
       "\n",
       "   col_48   col_49  col_50  \n",
       "0  5556.0   6514.0     225  \n",
       "1  4208.0   9019.0     225  \n",
       "2  3836.0  12437.0     225  \n",
       "3  4830.0   8588.0     225  \n",
       "4  4205.0  12931.0     225  \n",
       "\n",
       "[5 rows x 51 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "final.to_csv(\"./Dataset/final.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}