LR_Xin.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#libraries for modeling\n",
    "from multiprocessing.pool import ThreadPool\n",
    "from pyspark import SparkContext\n",
    "from pyspark.sql import SQLContext, SparkSession, Window, Row\n",
    "from pyspark.ml.linalg import Vectors\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "import pyspark.sql.functions as F\n",
    "import itertools\n",
    "from itertools import repeat\n",
    "import pickle\n",
    "import pyspark\n",
    "import copy\n",
    "\n",
    "from pyspark.ml.classification import MultilayerPerceptronClassifier\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "from pyspark.ml.classification import LogisticRegression\n",
    "from pyspark.ml.classification import GBTClassifier\n",
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
    "\n",
    "#libraries for plotting\n",
    "import matplotlib.gridspec as gridspec\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "class CreateBestModel:\n",
    "    def __init__(self, algo, avgprecision, avgrecall, avgfscore, hyperparams, ootmodel, ootprecision, ootrecall, ootfscore):\n",
    "        self.algo = algo\n",
    "        self.gsPrecision = avgprecision\n",
    "        self.gsFScore = avgfscore\n",
    "        self.gsRecall = avgrecall\n",
    "        self.hyperParams = hyperparams\n",
    "        self.model = ootmodel\n",
    "        self.ootPrecision = ootprecision\n",
    "        self.ootFScore = ootfscore\n",
    "        self.ootRecall = ootrecall\n",
    "\n",
    "#function-based\n",
    "def sample(df, sampling_method, ratio):\n",
    "\n",
    "    notfraud = df.select('*').where(df.Class == 0.0)\n",
    "    fraud = df.select('*').where(df.Class == 1.0)\n",
    "\n",
    "    if sampling_method == \"over\":\n",
    "        nrows = notfraud.select(\"Class\").count()\n",
    "        sample_size = int(nrows*ratio/(1-ratio))\n",
    "        sampled = fraud.rdd.takeSample(True, sample_size, 46)\n",
    "        fraud = sqlContext.createDataFrame(sampled)\n",
    "\n",
    "    elif sampling_method == \"under\":\n",
    "        nrows = fraud.select(\"Class\").count()\n",
    "        sample_size = int(nrows*(1-ratio)/ratio)\n",
    "        sampled = notfraud.rdd.takeSample(False, sample_size, 46)\n",
    "        notfraud = sqlContext.createDataFrame(sampled)\n",
    "    else:\n",
    "        return df\n",
    "    \n",
    "    sampled = fraud.union(notfraud)\n",
    "    \n",
    "    #shuffle undersampled dataframe\n",
    "    nrows = sampled.select(\"Class\").count()\n",
    "    shuffled = sampled.rdd.takeSample(False, nrows)\n",
    "    shuffled_df = sqlContext.createDataFrame(shuffled)\n",
    "\n",
    "    return shuffled_df\n",
    "\n",
    "def generateParamGrid(*args):\n",
    "    \n",
    "    grid = list(itertools.product(*args))\n",
    "    return grid\n",
    "\n",
    "def generateClassifier(algo, params, features):\n",
    "\n",
    "    ############################################################################\n",
    "    #TODO: complete this section\n",
    "\n",
    "    def lr(params,features):\n",
    "        print(params)\n",
    "        if len(params) > 2:\n",
    "            lrClassifier = LogisticRegression(featuresCol = 'features',\n",
    "                                          labelCol = 'Class',\n",
    "                                          threshold=params[0],\n",
    "                                          #regParam=params[0],\n",
    "                                           #elasticNetParam=params[0],\n",
    "                                           maxIter=params[1],\n",
    "                                           \n",
    "                                           weightCol=params[2])\n",
    "                                          #regParam=params[2])\n",
    "                                          \n",
    "        else:\n",
    "            lrClassifier = LogisticRegression(featuresCol = 'features',\n",
    "                                          labelCol = 'Class',\n",
    "                                              #regParam=params[0],\n",
    "                                            threshold=params[0],\n",
    "                                          #elasticNetParam=params[0],\n",
    "                                           maxIter=params[1])\n",
    "                                           \n",
    "        return lrClassifier\n",
    "\n",
    "\n",
    "    def gbm(params,features):\n",
    "        gbmClassifier = GBTClassifier(featuresCol = 'features',\n",
    "                                      labelCol = 'Class',\n",
    "                                      maxDepth = params[0],\n",
    "                                      minInfoGain = params[1])\n",
    "        return gbmClassifier\n",
    "\n",
    "    def rf(params,features):\n",
    "        rfClassifier = RandomForestClassifier(featuresCol='features',\n",
    "                                              labelCol='Class',\n",
    "                                              maxDepth=params[0],\n",
    "                                              minInfoGain=params[1],\n",
    "                                              numTrees=params[2])\n",
    "\n",
    "        return rfClassifier\n",
    "\n",
    "    def mlp(params,features):\n",
    "        input_layers = len(features)\n",
    "        layers = [input_layers, params[1], 2]\n",
    "        print(layers)\n",
    "        mlpClassifier = MultilayerPerceptronClassifier(featuresCol = 'features',\n",
    "                                                       labelCol = 'Class',\n",
    "                                                       maxIter = params[0],\n",
    "                                                       layers = layers,\n",
    "                                                       stepSize = params[2])\n",
    "        return mlpClassifier\n",
    "\n",
    "    def svm(params, features):\n",
    "        svmClassifier = LinearSVC(featuresCol = 'features',\n",
    "                         labelCol='Class', \n",
    "                         standardization=True,\n",
    "                         maxIter=params[0],\n",
    "                         regParam=params[1],\n",
    "                         tol =params[2]\n",
    "                         )\n",
    "        \n",
    "        return svmClassifier\n",
    "\n",
    "    def xg(params,features):\n",
    "        return\n",
    "    ############################################################################\n",
    "\n",
    "    getClassifier = {\n",
    "        'lr':lr,\n",
    "        'gbm':gbm,\n",
    "        'rf':rf,\n",
    "        'mlp':mlp,\n",
    "        'svm':svm,\n",
    "        'xg':xg}\n",
    "\n",
    "    return getClassifier[algo](params,features)\n",
    "\n",
    "def crossValidate(df, folds, k, classifier, features, sampling_method, ratio, pool):\n",
    "\n",
    "    def build(fold, df, classifier, features, sampling_method, ratio):\n",
    "\n",
    "        #undersample notfraud\n",
    "        validation = fold\n",
    "        train = df.subtract(fold)\n",
    "\n",
    "        #add class weight\n",
    "        #notfraud_count = train.select(\"Class\").where(train.Class == 0.0).count()\n",
    "        #total_count = train.select(\"Class\").count()\n",
    "        #balance_ratio = notfraud_count / total_count\n",
    "        balance_ratio = 0.7\n",
    "        train=train.withColumn(\"classWeights\", F.when(train.Class == 1.0,balance_ratio).otherwise(1-balance_ratio))\n",
    "        \n",
    "        train = sample(train, sampling_method, ratio)\n",
    "        \n",
    "        fraud_count = train.select(\"Class\").where(train.Class == 1.0).count()\n",
    "        tot_count = train.select(\"Class\").count()\n",
    "        fraud_ratio = fraud_count / tot_count\n",
    "\n",
    "        print(\"train: \" + str(tot_count))\n",
    "        print(\"fraud ratio: \" + str(fraud_ratio))\n",
    "\n",
    "        vectorAssembler = VectorAssembler(inputCols = features, outputCol = 'features')\n",
    "        vector_train = vectorAssembler.transform(train)\n",
    "        vector_validate = vectorAssembler.transform(validation)\n",
    "        model = classifier.fit(vector_train)\n",
    "        pred = model.transform(vector_validate)\n",
    "        pos = pred.filter(pred.prediction == 1.0).count()\n",
    "        if pos != 0:\n",
    "            precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pos\n",
    "        else:\n",
    "            precision = 0\n",
    "        fraud = pred.filter(pred.Class == 1.0).count()\n",
    "        if fraud != 0:\n",
    "            recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / fraud\n",
    "        else:\n",
    "            recall = 0\n",
    "        precision_recall = precision + recall\n",
    "        if precision_recall != 0:\n",
    "            f_score = 2 * precision * recall /(precision_recall)\n",
    "        else:\n",
    "            f_score = 0\n",
    "        print(\"\\n precision, recall, f_score: \" + str(precision) + \", \" + str(recall) + \", \" + str(f_score))\n",
    "        return [precision, recall, f_score]\n",
    "\n",
    "    #call multiprocessing here\n",
    "    cvperformance = pool.map(lambda fold: build(fold, df, classifier, features, sampling_method, ratio), folds)\n",
    "\n",
    "    #calculate metrics\n",
    "    precision_sum = sum([x[0] for x in cvperformance])\n",
    "    recall_sum = sum([x[1] for x in cvperformance])\n",
    "\n",
    "    avg_precision = precision_sum/k\n",
    "    avg_recall = recall_sum/k\n",
    "    if avg_precision+avg_recall == 0:\n",
    "        avg_fscore = 0\n",
    "    else:\n",
    "        avg_fscore = 2 * avg_precision * avg_recall /(avg_precision+avg_recall)\n",
    "    return [avg_precision,avg_recall,avg_fscore]\n",
    "\n",
    "\n",
    "def gridSearch(df, folds, k, algo, grid, features, sampling_method, ratio, pool):\n",
    "\n",
    "    best_hyper = None\n",
    "    best_precision = 0\n",
    "    best_recall = 0\n",
    "    best_fscore = 0\n",
    "\n",
    "    for i in range(len(grid)):\n",
    "        params = list(grid[i])\n",
    "        classifier = generateClassifier(algo, params, features)\n",
    "        modelPerformance = crossValidate(df, folds, k, classifier, features, sampling_method, ratio, pool)\n",
    "        if modelPerformance[2] > best_fscore:\n",
    "            best_hyper = params\n",
    "            best_precision = modelPerformance[0]\n",
    "            best_recall = modelPerformance[1]\n",
    "            best_fscore = modelPerformance[2]\n",
    "\n",
    "    return best_hyper, best_precision, best_recall, best_fscore\n",
    "\n",
    "def ootTest(traindf,testdf, algo,features,params):\n",
    "    vectorAssembler = VectorAssembler(inputCols = features, outputCol = 'features')\n",
    "    classifier = generateClassifier(algo, params, features)\n",
    "    vector_train = vectorAssembler.transform(traindf)\n",
    "    vector_test = vectorAssembler.transform(testdf)\n",
    "    ootmodel = classifier.fit(vector_train)\n",
    "    pred = ootmodel.transform(vector_test)\n",
    "    pos = pred.filter(pred.prediction == 1.0).count()\n",
    "    if pos != 0:\n",
    "        precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pos\n",
    "    else:\n",
    "        precision = 0\n",
    "    fraud = pred.filter(pred.Class == 1.0).count()\n",
    "    if fraud != 0:\n",
    "        recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / fraud\n",
    "    else:\n",
    "        recall = 0\n",
    "    precision_recall = precision + recall\n",
    "    if precision_recall != 0:\n",
    "        f_score = 2 * precision * recall /(precision_recall)\n",
    "    else:\n",
    "        f_score = 0\n",
    "    print(\"\\n precision, recall, f_score: \" + str(precision) + \", \" + str(recall) + \", \" + str(f_score))\n",
    "    \n",
    "    return ootmodel, precision, recall, f_score\n",
    "\n",
    "def tune(df, k, stratification_flag, sampling_method, ratio, modelobj_flag, features, algo, *args, **kwargs):\n",
    "\n",
    "    \"\"\"\n",
    "    Entry point of this suite of functions. returns cv metrics or a model object\n",
    "    Example:\n",
    "        >>> cv_hyper, cv_precision, cv_recall, cv_fscore = tune(df, 5, True,\n",
    "        'None', 0, False, features, 'mlp', [100], [15], [0.03])\n",
    "    :param df: data for modeling purpose\n",
    "    :type df: : pyspark dataframe\n",
    "    :param k: number of folds for cross validation\n",
    "    :type k: int\n",
    "    :param stratification_flag: specifies whether fraud ratio is fixed for each fold. True for stratification\n",
    "    :type stratification_flag: boolean\n",
    "    :param sampling_method: \"over\" for oversampling minority class, \"under\" for undersampling majority class, \"None\"\n",
    "    :type sampling_method: str\n",
    "    :param ratio: targeted fraud ratio after sampling.\n",
    "    :type ratio: float\n",
    "    :param modelobj_flag: specifies whether to return a model object for out of time test. if False, returns cv performancce\n",
    "    :type modelobj_flag: float\n",
    "    :param features: features for training\n",
    "    :type features: list\n",
    "    :param *args: a sequence of params for hyperparams tuning. ex. [values for params1], [values for params2],...\n",
    "    :type *args: list\n",
    "    :returns: model object or cross validation metrics depending on modelobj_flag\n",
    "    \"\"\"\n",
    "\n",
    "\n",
    "    pool = ThreadPool(2)\n",
    "\n",
    "    #reduce df dimenions to include features and class\n",
    "    cols = features+['Class', 'index']\n",
    "    df = df.select(cols)\n",
    "    df = df.select(*(F.col(c).cast(\"double\").alias(c) for c in df.columns))\n",
    "    #df.drop(\"index\")\n",
    "\n",
    "    folds = []\n",
    "    \n",
    "    ########################ClassWeights#################################\n",
    "    if algo in [\"lr\", \"svm\"] and [\"ClassWeigts\"] in args:\n",
    "        #add class weight\n",
    "        balance_ratio = args[-1][0]\n",
    "        df=df.withColumn(\"classWeights\", when(df.Class == 1.0,balance_ratio).otherwise(1-balance_ratio))\n",
    "    ########################ClassWeights#################################\n",
    "    \n",
    "    if stratification_flag == False:\n",
    "\n",
    "        tot_count = df.select(\"Class\").count()\n",
    "        n = int(tot_count / k)\n",
    "\n",
    "        #create sub-dataframe iteratively\n",
    "        fold_start = 1\n",
    "        fold_end = n\n",
    "        for i in range(k):\n",
    "            fold = df.select('*').where(df.index.between(fold_start, fold_end))\n",
    "            folds.append(fold)\n",
    "            fold_start = fold_end + 1\n",
    "            fold_end = fold_start + n\n",
    "            if i == k-2:\n",
    "                end = tot_count\n",
    "\n",
    "    #ensure each fold has the same number of records and same fraud ratio\n",
    "    if stratification_flag == True:\n",
    "\n",
    "        fraud = df.select(\"*\").where(df.Class == 1.0)\n",
    "        #shuffle undersampled dataframe\n",
    "        nrows = fraud.select(\"Class\").count()\n",
    "        shuffled = fraud.rdd.takeSample(False, nrows)\n",
    "        fraud = sqlContext.createDataFrame(shuffled)\n",
    "        #add row index to dataframe\n",
    "        fraud = fraud.withColumn('dummy', F.lit('7'))\n",
    "        fraud = fraud.withColumn(\"temp_index\", F.row_number().over(Window.partitionBy(\"dummy\").orderBy(\"dummy\")))\n",
    "        fraud = fraud.drop('dummy')\n",
    "        fraud_count = fraud.select(\"Class\").count()\n",
    "        each_fraud = int(fraud_count/k)\n",
    "\n",
    "        notfraud = df.select(\"*\").where(df.Class == 0.0)\n",
    "        nrows = notfraud.select(\"Class\").count()\n",
    "        shuffled = notfraud.rdd.takeSample(False, nrows)\n",
    "        notfraud = sqlContext.createDataFrame(shuffled)\n",
    "        #add row index to dataframe\n",
    "        notfraud = notfraud.withColumn('dummy', F.lit('7'))\n",
    "        notfraud = notfraud.withColumn(\"temp_index\", F.row_number().over(Window.partitionBy(\"dummy\").orderBy(\"dummy\")))\n",
    "        notfraud = notfraud.drop('dummy')\n",
    "        notfraud_count = notfraud.select(\"Class\").count()\n",
    "        each_notfraud = int(notfraud_count/k)\n",
    "\n",
    "        fraud_start = 1\n",
    "        fraud_end = each_fraud\n",
    "        notfraud_start = 1\n",
    "        notfraud_end = each_notfraud\n",
    "\n",
    "        for i in range(k):\n",
    "            fraud_fold  = fraud.select('*').where(fraud.temp_index.between(fraud_start, fraud_end))\n",
    "            notfraud_fold = notfraud.select('*').where(notfraud.temp_index.between(notfraud_start, notfraud_end))\n",
    "            fold = fraud_fold.union(notfraud_fold).drop(\"temp_index\")\n",
    "            folds.append(fold)\n",
    "            fraud_start = fraud_end + 1\n",
    "            fraud_end = fraud_start + each_fraud\n",
    "            notfraud_start = notfraud_end + 1\n",
    "            notfraud_end = notfraud_start + each_notfraud\n",
    "            if i == k-2:\n",
    "                fraud_end = fraud_count\n",
    "                notfraud_end = notfraud_count\n",
    "\n",
    "    #generate hyperparam combo\n",
    "    grid = generateParamGrid(*args)\n",
    "\n",
    "    #conduct grid search:\n",
    "    best_hyper, best_precision, best_recall, best_fscore = gridSearch(df, folds, k, algo, grid, features, sampling_method, ratio, pool)\n",
    "\n",
    "    if modelobj_flag == True:\n",
    "        #generate a model obj\n",
    "        traindf = sample(df, sampling_method, ratio)\n",
    "        testdf = sqlContext.read.csv(\"oot.csv\", header = True)\n",
    "        cols = features+['Class', 'index']\n",
    "        testdf = testdf.select(cols)\n",
    "        testdf = testdf.select(*(F.col(c).cast(\"double\").alias(c) for c in testdf.columns))\n",
    "        model, precision, recall, fscore = ootTest(traindf, testdf, algo,features,best_hyper)\n",
    "        \n",
    "        modelobj = CreateBestModel(algo, best_precision, best_recall, best_fscore, best_hyper, \n",
    "                                   model, precision, recall, fscore)\n",
    "        return modelobj\n",
    "\n",
    "    return best_hyper, best_precision, best_recall, best_fscore\n",
    "\n",
    "def save(modelobj, filename):\n",
    "\n",
    "    modelobj = modelobj\n",
    "    pickle.dump(modelobj, open(filename, \"wb\"))\n",
    "\n",
    "def load(filename):\n",
    "\n",
    "    modelobj = pickle.load(open(filename, \"rb\"))\n",
    "    return modelobj\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## i) Base Model: Train base_train and test base_test using default params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train before sampling: 181584\n",
      "fraud ratio: 0.0016961846858754076\n",
      "test: 45396\n",
      "+-----+------------+\n",
      "|Class|count(Class)|\n",
      "+-----+------------+\n",
      "|  0.0|       45305|\n",
      "|  1.0|          91|\n",
      "+-----+------------+\n",
      "\n",
      "+----------+-----------------+\n",
      "|prediction|count(prediction)|\n",
      "+----------+-----------------+\n",
      "|       0.0|            45335|\n",
      "|       1.0|               61|\n",
      "+----------+-----------------+\n",
      "\n",
      "Precision is  0.8852459016393442\n",
      "Recall is  0.5934065934065934\n",
      "F1 score is  0.7105263157894737\n",
      "Test Area Under ROC 0.9775965343563068\n"
     ]
    }
   ],
   "source": [
    "# Base Model: Train base_train and test base_test using default params\n",
    "sc=pyspark.SparkContext.getOrCreate()\n",
    "sqlContext = SQLContext(sc)\n",
    "sqlContext.clearCache()\n",
    "\n",
    "trainData = sqlContext.read.csv(\"base_train.csv\", header = True)\n",
    "testData = sqlContext.read.csv(\"base_test.csv\", header = True)\n",
    "trainData = trainData.select(*(F.col(c).cast(\"double\").alias(c) for c in trainData.columns))\n",
    "testData = testData.select(*(F.col(c).cast(\"double\").alias(c) for c in testData.columns))\n",
    "train_count = trainData.select(\"Class\").count()\n",
    "train_fraud_count = trainData.select(\"Class\").where(trainData.Class == 1).count()\n",
    "test_count = testData.select(\"Class\").count()\n",
    "print(\"train before sampling: \" + str(train_count))\n",
    "print(\"fraud ratio: \" + str(train_fraud_count/train_count))\n",
    "print(\"test: \" + str(test_count))\n",
    "\n",
    "features = ['Amount', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28']\n",
    "vectorAssembler = VectorAssembler(inputCols = features, outputCol = 'features')\n",
    "df_train = vectorAssembler.transform(trainData)\n",
    "df_test = vectorAssembler.transform(testData)\n",
    "\n",
    "\n",
    "lr = LogisticRegression(featuresCol = 'features', labelCol='Class')\n",
    "# Fit the model\n",
    "lrModel = lr.fit(df_train)\n",
    "# predict the model\n",
    "pred = lrModel.transform(df_test)\n",
    "pred = pred.select(\"features\", \"Class\", \"rawPrediction\", \"prediction\")\n",
    "\n",
    "#evaluate\n",
    "pred.groupby('Class').agg({'Class': 'count'}).show()\n",
    "pred.groupby('prediction').agg({'prediction': 'count'}).show()\n",
    "precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.prediction == 1.0).count()\n",
    "recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.Class == 1.0).count()\n",
    "f1 = 2 * precision * recall /(precision + recall)\n",
    "\n",
    "\n",
    "print(\"Precision is \", precision)\n",
    "print(\"Recall is \", recall)\n",
    "print(\"F1 score is \", f1)\n",
    "evaluator = BinaryClassificationEvaluator(\n",
    "    labelCol=\"Class\", metricName=\"areaUnderROC\")\n",
    "print('Test Area Under ROC', evaluator.evaluate(pred))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step2. Comparing random and stratified cv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5-fold random cv with default param and sample_method=None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.5, 100]\n",
      "train: 152562\n",
      "fraud ratio: 0.0014420366801693738\n",
      "train: 152482\n",
      "fraud ratio: 0.0016592122348867407\n",
      "\n",
      " precision, recall, f_score: 0.7011494252873564, 0.6931818181818182, 0.6971428571428572\n",
      "\n",
      " precision, recall, f_score: 0.9411764705882353, 0.5818181818181818, 0.7191011235955057\n",
      "train: 152538\n",
      "fraud ratio: 0.0017372720240202441\n",
      "train: 152553\n",
      "fraud ratio: 0.0018223174896593315\n",
      "\n",
      " precision, recall, f_score: 1.0, 0.32558139534883723, 0.49122807017543857\n",
      "\n",
      " precision, recall, f_score: 0.8823529411764706, 0.5, 0.6382978723404256\n",
      "train: 152499\n",
      "fraud ratio: 0.0016655846923586384\n",
      "\n",
      " precision, recall, f_score: 0.90625, 0.5370370370370371, 0.6744186046511628\n",
      "grid search precision: 0.8861857674104124\n",
      "grid search recall: 0.5275236864771748\n",
      "grid search f-score: 0.6613579355254386\n"
     ]
    }
   ],
   "source": [
    "df = sqlContext.read.csv(\"base_train.csv\", header = True)\n",
    "features = ['V1','V2','V3','V4','V5','V6','V7','V8','V9',\n",
    "                                            'V10','V11','V12','V13','V14','V15','V16','V17','V18',\n",
    "                                            'V19','V20','V21','V22','V23','V24','V25','V26','V27',\n",
    "                                            'V28','Amount']\n",
    "gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, False, 'None', 0, False, features, 'lr', [0.5],[100])\n",
    "print(\"grid search precision:\", gs_precision)\n",
    "print(\"grid search recall:\", gs_recall)\n",
    "print(\"grid search f-score:\", gs_fscore)\n",
    "sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5-fold stratified cv with default param and sample_method=None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.5, 100]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8947368421052632, 0.5573770491803278, 0.686868686868687\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8823529411764706, 0.4838709677419355, 0.625\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.76, 0.6129032258064516, 0.6785714285714285\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8292682926829268, 0.5483870967741935, 0.6601941747572816\n",
      "\n",
      " precision, recall, f_score: 0.8604651162790697, 0.6065573770491803, 0.7115384615384615\n",
      "grid search precision: 0.845364638448746\n",
      "grid search recall: 0.5618191433104178\n",
      "grid search f-score: 0.6750248874592001\n"
     ]
    }
   ],
   "source": [
    "#stratified 5cv with default param(threshold version)\n",
    "df = sqlContext.read.csv(\"base_train.csv\", header = True)\n",
    "features = ['Amount', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28']\n",
    "gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None', 0, False, features, 'lr', [0.5],[100])\n",
    "print(\"grid search precision:\", gs_precision)\n",
    "print(\"grid search recall:\", gs_recall)\n",
    "print(\"grid search f-score:\", gs_fscore)\n",
    "sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step3. stratified cv grid search "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.2, 110]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.7719298245614035, 0.7213114754098361, 0.7457627118644067\n",
      "\n",
      " precision, recall, f_score: 0.782608695652174, 0.5806451612903226, 0.6666666666666667\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8070175438596491, 0.7419354838709677, 0.7731092436974789\n",
      "\n",
      " precision, recall, f_score: 0.8235294117647058, 0.6774193548387096, 0.743362831858407\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8125, 0.639344262295082, 0.7155963302752295\n",
      "[0.2, 150]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.7586206896551724, 0.7213114754098361, 0.7394957983193277\n",
      "\n",
      " precision, recall, f_score: 0.782608695652174, 0.5806451612903226, 0.6666666666666667\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8070175438596491, 0.7419354838709677, 0.7731092436974789\n",
      "\n",
      " precision, recall, f_score: 0.8269230769230769, 0.6935483870967742, 0.7543859649122807\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8125, 0.639344262295082, 0.7155963302752295\n",
      "[0.2, 200]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.7586206896551724, 0.7213114754098361, 0.7394957983193277\n",
      "\n",
      " precision, recall, f_score: 0.782608695652174, 0.5806451612903226, 0.6666666666666667\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8070175438596491, 0.7419354838709677, 0.7731092436974789\n",
      "\n",
      " precision, recall, f_score: 0.8235294117647058, 0.6774193548387096, 0.743362831858407\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8125, 0.639344262295082, 0.7155963302752295\n",
      "[0.3, 110]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8076923076923077, 0.6885245901639344, 0.743362831858407\n",
      "\n",
      " precision, recall, f_score: 0.7906976744186046, 0.5483870967741935, 0.6476190476190475\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8, 0.7096774193548387, 0.7521367521367521\n",
      "\n",
      " precision, recall, f_score: 0.851063829787234, 0.6451612903225806, 0.7339449541284404\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8222222222222222, 0.6065573770491803, 0.6981132075471698\n",
      "[0.3, 150]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.7962962962962963, 0.7049180327868853, 0.7478260869565216\n",
      "\n",
      " precision, recall, f_score: 0.7674418604651163, 0.532258064516129, 0.6285714285714286\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.851063829787234, 0.6451612903225806, 0.7339449541284404\n",
      "\n",
      " precision, recall, f_score: 0.8148148148148148, 0.7096774193548387, 0.7586206896551724\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8222222222222222, 0.6065573770491803, 0.6981132075471698\n",
      "[0.3, 200]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "\n",
      " precision, recall, f_score: 0.7674418604651163, 0.532258064516129, 0.6285714285714286\n",
      "\n",
      " precision, recall, f_score: 0.7962962962962963, 0.7049180327868853, 0.7478260869565216\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8541666666666666, 0.6612903225806451, 0.7454545454545455\n",
      "\n",
      " precision, recall, f_score: 0.8301886792452831, 0.7096774193548387, 0.7652173913043478\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8043478260869565, 0.6065573770491803, 0.691588785046729\n",
      "[0.4, 110]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8297872340425532, 0.639344262295082, 0.7222222222222223\n",
      "\n",
      " precision, recall, f_score: 0.8048780487804879, 0.532258064516129, 0.6407766990291263\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8367346938775511, 0.6612903225806451, 0.7387387387387386\n",
      "\n",
      " precision, recall, f_score: 0.8636363636363636, 0.6129032258064516, 0.7169811320754716\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8571428571428571, 0.5901639344262295, 0.6990291262135924\n",
      "[0.4, 150]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8048780487804879, 0.532258064516129, 0.6407766990291263\n",
      "\n",
      " precision, recall, f_score: 0.8297872340425532, 0.639344262295082, 0.7222222222222223\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8367346938775511, 0.6612903225806451, 0.7387387387387386\n",
      "\n",
      " precision, recall, f_score: 0.8604651162790697, 0.5967741935483871, 0.7047619047619047\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8536585365853658, 0.5737704918032787, 0.6862745098039217\n",
      "[0.4, 200]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8260869565217391, 0.6229508196721312, 0.7102803738317758\n",
      "\n",
      " precision, recall, f_score: 0.8048780487804879, 0.532258064516129, 0.6407766990291263\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8571428571428571, 0.6774193548387096, 0.7567567567567567\n",
      "\n",
      " precision, recall, f_score: 0.8636363636363636, 0.6129032258064516, 0.7169811320754716\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.85, 0.5573770491803278, 0.6732673267326733\n",
      "grid search precision: 0.7975340012180144\n",
      "grid search recall: 0.6753569539925965\n",
      "grid search f-score: 0.7313781537766424\n",
      "grid search hyper: [0.2, 150]\n"
     ]
    }
   ],
   "source": [
    "#grid search with threshold [0.2,150] f1=0.73\n",
    "df = sqlContext.read.csv(\"base_train.csv\", header = True)\n",
    "features = ['V1','V2','V3','V4','V5','V6','V7','V8','V9',\n",
    "                                            'V10','V11','V12','V13','V14','V15','V16','V17','V18',\n",
    "                                            'V19','V20','V21','V22','V23','V24','V25','V26','V27',\n",
    "                                            'V28','Amount']\n",
    "gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None', 0, False, features, 'lr', [0.2,0.3,0.4],[110,150,200])\n",
    "print(\"grid search precision:\", gs_precision)\n",
    "print(\"grid search recall:\", gs_recall)\n",
    "print(\"grid search f-score:\", gs_fscore)\n",
    "print(\"grid search hyper:\", gs_hyper)\n",
    "sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Best hyper parameter in first grid search\n",
    "#### grid search hyper: [0.2, 150]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# use best param in step3 test base_test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+------------+\n",
      "|Class|count(Class)|\n",
      "+-----+------------+\n",
      "|  0.0|       45305|\n",
      "|  1.0|          91|\n",
      "+-----+------------+\n",
      "\n",
      "+----------+-----------------+\n",
      "|prediction|count(prediction)|\n",
      "+----------+-----------------+\n",
      "|       0.0|            45317|\n",
      "|       1.0|               79|\n",
      "+----------+-----------------+\n",
      "\n",
      "Precision is  0.8227848101265823\n",
      "Recall is  0.7142857142857143\n",
      "F1 score is  0.7647058823529411\n"
     ]
    }
   ],
   "source": [
    "lr = LogisticRegression(featuresCol = 'features', labelCol='Class',threshold=0.2, maxIter=150)\n",
    "lrModel = lr.fit(df_train)\n",
    "pred = lrModel.transform(df_test)\n",
    "pred = pred.select(\"features\", \"Class\", \"rawPrediction\", \"prediction\")\n",
    "pred.groupby('Class').agg({'Class': 'count'}).show()\n",
    "pred.groupby('prediction').agg({'prediction': 'count'}).show()\n",
    "precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.prediction == 1.0).count()\n",
    "recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.Class == 1.0).count()\n",
    "f1 = 2 * precision * recall /(precision + recall)\n",
    "print(\"Precision is \", precision)\n",
    "print(\"Recall is \", recall)\n",
    "print(\"F1 score is \", f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### result after first grid search\n",
    "#### Precision is  0.8227848101265823\n",
    "#### Recall is  0.7142857142857143\n",
    "#### F1 score is  0.7647058823529411"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step4. use best param from 3) for feature coefficients\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3kAAAIOCAYAAAAFoMQYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xm4JGdZN+DfQxII+zosghIMCCJCokNEUJQgEPawGgQERcMmBlFk+VBxR0CRRcGwRmQnrAECiCEQQXCyECBhJ0ggkEGDCSIBkuf7o2rMcZztTE6dnqm+7+vq63RXd7/PW9V16vTvvLVUdwcAAIB5uMyiOwAAAMDaEfIAAABmRMgDAACYESEPAABgRoQ8AACAGRHyAAAAZkTIA2DdVNUBVdVVtXHRfdldVXWzqvpwVX2nqs7awbSuqvvvYpsPr6pvTddrAJbJvovuAADsZf4kybeT3CzJf+1g2vWSnLeLbb4uyTvXsI9JhqCZ5AHd/ca1bhuAPZeQBwCrc+Mkb+3us3Y0rbu/tqsNdvd/J/nvteogAMvN7poArKmqOqyqPlhV51XVf1TVu6vqR7d62Y9U1Unj7o2fqqo7r3j/flX1vKr6alVdWFVfrqpn7GLty1bVn1XVl8b3fqGqfnPF87evqo+Mdb9eVc+pqsuueL6q6ner6vNV9d9V9fGqesiK5zvJrZL8/rg75tO3NW3La1furllVP1BVr6qqf6+qb1fVaVV1h/G5/7O7ZlXds6pOHvv6xar60636elZVPa2q/q6qzq+qs6vqiSufH+++YezLWeP0H6yqt46fzbfH5X/ErixfAPYORvIAWGtXTPLXSU5PcvkkT0vy9qq6+YrXPDPJE8bXPDbJW6vqxt39lSS/meQ+SY5IclaSGyS56S7WPibJzyY5KsmpSW6Y5AeTpKqun+RdSV6Z5OFJDkzykiQXJ/nt8f1/kuT+Y58+neSnk7y4qs7r7ndk2AXz/UmOS/LsJN9K8qJtTPtfquqKSU5Mcu44b1/JEAy3qarukuRV43x8IMkPjXUul+R3Vrz0t5L8QZJnJblrkudV1Und/eEktx7r/frYt4vG9/xtkv2T3CHJ+dn1ZQvAXkLIA2BNdfexKx9X1a9kCBOHJDl7nPzC7n79+PxRSe6S5NEZAuENk3wmyQe7u5P8W5IP7axuVd0kQzC8a3cfP07+woqXPCbJOUke090XJzmzqp6c5O+q6veSVIbgeefu/uD4ni9W1SEZQt87uvtrVfX9JN9asTvmt7YxbWu/lOS6SX66u78xTvv8Dmbn/yV5Vne/fMtrq+pJSf6hqp44LpckeU93v2C8//xx1PKOST7c3ZurKkm+uVW/bpjk2O7+2JZ53EE/ANgLCXkArKmqOjDJHyf5qSQbMhwacJkMo1FbQt6Ht7y+uy+uqo8k2TLS94ok703ymap6T4YTkrxrDGY7cnCGUbkTtvP8j2YIPyvbOSnJZTMcU3e5DCNcx4+7YG6xX4YRxUvj4CSnrwh4O/OTSQ4Zg90Wl8kwMnrdDGE1GUZCV/pqkmvvpO3nJnlRVR2W5H1J3tzdJ+9ivwDYCwh5AKy1t2fYHfGR48/vJzkjQ5jaqe4+paoOSHJYkkMz7IL5saq6006CXu2k6UrS23muc8lx6vfMMHq40vd20vbO7KxvW7tMkj9M8oZtPLd5xf2t+7VyPrapu19aVe9Ocrckv5DkQ1X159399FX2EYA9lBOvALBmquqaGUbM/qy7/7G7z0xy5fzffyreZsV7KsOunGdumdbdF3T3G7r70UnuniHs3Xgn5U/J8HftDtt5/owkP11VK//2/UyS72bYdfKMJBcmuWF3f26r25d2UntnTklyy6q61ipef7Nt9ONz3f39VdT9XpJ9tp7Y3Wd399Hd/cAkv5/kyFW0CcAezkgeAGvpvCTfSPLrVfXlJNfPcFKQrYPJo6vqM0k+nuFYuRsmeWGSVNUTMuyOeFqGkPJLGY7pOzs70N2frarXJ3nJeJzfKRlO2nJAd78ywwlHHp/kb6vquUl+OMkzkrygu7891n52kmePwfMDSa6UIZBe3N1H7/ZSSV6d5MlJ3lJVTxnn5ceTXNDd29q99I+SHFdVX0ry+gzL7xZJDunu311F3bOS3LGqTkxyYXefN877uzIc93iVDCOmZ+zebAGwJzKSB8CaGXen/MUkt0zyiSR/k+T3MoyQrfTkDCc5+ViGkHGf7t4S4i5I8sQkH80Q1A7KcDKVb+9CF345Q6B6XpJPZTi+76pj376S4QyUB2cIkC9L8pokT13x/t9L8vQMZ7D8ZIZjA++XS3lyku7+ryQ/l2H31bePbf9htrP7aHe/O8MI5h0yLIePZlhmW+9GujO/Pbbx5QxnG02Gv/3PzxDs3pvk60ketsp2AdiD1SUn6AIAAGBvZyQPAABgRhyTB8Beoap+NsOxZNvU3Vdax+4AwB7L7poA7BWq6vIZTuSyTd39uXXsDgDssYQ8AACAGXFMHgAAwIzsNcfkXeta1+oDDjhg0d0AAABYiJNPPvkb3b1hZ6/ba0LeAQcckE2bNi26GwAAAAtRVV/aldfZXRMAAGBGhDwAAIAZEfIAAABmRMgDAACYESEPAABgRoQ8AACAGRHyAAAAZkTIAwAAmBEhDwAAYEaEPAAAgBkR8gAAAGZEyAMAAJgRIQ8AAGBGhDwAAIAZEfIAAABmRMgDAACYESEPAABgRoQ8AACAGRHyAAAAZmTfRXdgd2x+4T9M1vaGRz9ksrYBAACmZiQPAABgRoQ8AACAGRHyAAAAZkTIAwAAmBEhDwAAYEaEPAAAgBkR8gAAAGZEyAMAAJgRIQ8AAGBGhDwAAIAZEfIAAABmRMgDAACYESEPAABgRoQ8AACAGRHyAAAAZkTIAwAAmBEhDwAAYEaEPAAAgBkR8gAAAGZEyAMAAJgRIQ8AAGBG1iXkVdU+VXVqVR03Pr5RVX2kqj5bVa+rqsuuRz8AAADmbr1G8o5KcuaKx3+R5DndfZMk5yV5xDr1AwAAYNYmD3lVdYMkd0/ykvFxJTk0yRvHlxyT5PCp+wEAALAM1mMk76+T/G6Si8fH10zyze7+/vj47CTX39Ybq+rIqtpUVZs2b948fU8BAAD2cpOGvKq6R5Jzu/vklZO38dLe1vu7++ju3tjdGzds2DBJHwEAAOZk34nbv12Se1XV3ZLsn+QqGUb2rlZV+46jeTdI8tWJ+wEAALAUJh3J6+6ndPcNuvuAJEck+afufnCSE5Lcf3zZw5K8dcp+AAAALItFXSfvSUmeUFWfy3CM3ksX1A8AAIBZmXp3zf/R3e9P8v7x/heSHLJetQEAAJbFokbyAAAAmICQBwAAMCNCHgAAwIwIeQAAADMi5AEAAMyIkAcAADAjQh4AAMCMCHkAAAAzIuQBAADMiJAHAAAwI0IeAADAjAh5AAAAMyLkAQAAzIiQBwAAMCNCHgAAwIwIeQAAADMi5AEAAMyIkAcAADAjQh4AAMCMCHkAAAAzIuQBAADMiJAHAAAwI0IeAADAjAh5AAAAMyLkAQAAzIiQBwAAMCNCHgAAwIwIeQAAADMi5AEAAMyIkAcAADAj+y66A3uLc1/03MnavvajjpqsbQAAYLkYyQMAAJgRIQ8AAGBGhDwAAIAZEfIAAABmRMgDAACYESEPAABgRoQ8AACAGRHyAAAAZmTSkFdV+1fVR6vqY1X1yar6w3H6K6rqi1V12ng7aMp+AAAALIt9J27/wiSHdve3qmq/JCdV1bvG557Y3W+cuD4AAMBSmTTkdXcn+db4cL/x1lPWBAAAWGaTH5NXVftU1WlJzk3y3u7+yPjUn1bV6VX1nKq63NT9AAAAWAaTh7zuvqi7D0pygySHVNUtkjwlyc2S3DrJNZI8aVvvraojq2pTVW3avHnz1F0FAADY663b2TW7+5tJ3p/ksO4+pwcXJnl5kkO2856ju3tjd2/csGHDenUVAABgrzX12TU3VNXVxvuXT/ILST5VVdcbp1WSw5N8Ysp+AAAALIupz655vSTHVNU+GQLl67v7uKr6p6rakKSSnJbkURP3AwAAYClMfXbN05McvI3ph05ZFwAAYFmt2zF5AAAATE/IAwAAmBEhDwAAYEaEPAAAgBkR8gAAAGZEyAMAAJgRIQ8AAGBGhDwAAIAZEfIAAABmRMgDAACYESEPAABgRoQ8AACAGRHyAAAAZkTIAwAAmBEhDwAAYEaEPAAAgBkR8gAAAGZEyAMAAJgRIQ8AAGBGhDwAAIAZEfIAAABmRMgDAACYESEPAABgRoQ8AACAGRHyAAAAZkTIAwAAmBEhDwAAYEaEPAAAgBkR8gAAAGZEyAMAAJgRIQ8AAGBGhDwAAIAZEfIAAABmRMgDAACYESEPAABgRoQ8AACAGRHyAAAAZkTIAwAAmBEhDwAAYEaEPAAAgBmZNORV1f5V9dGq+lhVfbKq/nCcfqOq+khVfbaqXldVl52yHwAAAMti6pG8C5Mc2t23SnJQksOq6jZJ/iLJc7r7JknOS/KIifsBAACwFCYNeT341vhwv/HWSQ5N8sZx+jFJDp+yHwAAAMti8mPyqmqfqjotyblJ3pvk80m+2d3fH19ydpLrb+e9R1bVpqratHnz5qm7CgAAsNebPOR190XdfVCSGyQ5JMmPbutl23nv0d29sbs3btiwYcpuAgAAzMK6nV2zu7+Z5P1JbpPkalW17/jUDZJ8db36AQAAMGdTn11zQ1Vdbbx/+SS/kOTMJCckuf/4socleeuU/QAAAFgW++78JZfK9ZIcU1X7ZAiUr+/u46rqjCSvrao/SXJqkpdO3A8AAIClMGnI6+7Tkxy8jelfyHB8HgAAAGto3Y7JAwAAYHpT767JbvrK3/zmpO1f/7HPm7R9AABgMYzkAQAAzIiQBwAAMCN21+R/nPG395qs7Zs/5m2TtQ0AAFzCSB4AAMCMCHkAAAAzIuQBAADMiJAHAAAwI0IeAADAjAh5AAAAMyLkAQAAzIiQBwAAMCNCHgAAwIwIeQAAADMi5AEAAMyIkAcAADAjQh4AAMCMCHkAAAAzIuQBAADMiJAHAAAwI0IeAADAjAh5AAAAMyLkAQAAzIiQBwAAMCNCHgAAwIwIeQAAADMi5AEAAMyIkAcAADAjQh4AAMCMCHkAAAAzssshr6qOqqqr1OClVXVKVd15ys4BAACwOqsZyfvV7j4/yZ2TbEjyK0meMUmvAAAA2C2rCXk1/rxbkpd398dWTAMAAGAPsJqQd3JVvSdDyHt3VV05ycXTdAsAAIDdse8qXvuIJAcl+UJ3f7uqrplhl00AAAD2EKsZyXtvd5/S3d9Mku7+9yTPmaZbAAAA7I6djuRV1f5JrpDkWlV19VxyHN5VkvzAhH0DAABglXZld81HJnl8hkB3ci4Jeecn+ZuJ+gUAAMBu2Onumt393O6+UZLf6e4f7u4bjbdbdfcLdvTeqvrBqjqhqs6sqk9W1VHj9KdX1Veq6rTxdrc1mh8AAICltssnXunu51fVbZMcsPJ93f33O3jb95P8dnefMp6N8+Sqeu/43HO6+9m70WcAAAC2Y5dDXlW9MsmBSU5LctE4uZNsN+R19zlJzhnvX1BVZya5/m73FgAAgB1azSUUNia5eXf37hSqqgOSHJzkI0lul+Q3quqXk2zKMNp33u60y97tn4++x2Rt3+7I4yZrGwAA9lSruYTCJ5Jcd3eKVNWVkhyb5PHdfX6SF2YYFTwow0jfX27nfUdW1aaq2rR58+bdKQ0AALBUVjOSd60kZ1TVR5NcuGVid99rR2+qqv0yBLxXdfebxvd8fcXzL06yzSGX7j46ydFJsnHjxt0aQQQAAFgmqwl5T19t41VVSV6a5Mzu/qsV0683Hq+XJPfJMEoIAADApbSas2ueWFU3THKT7v7HqrpCkn128rbbJXloko9X1WnjtKcmeVBVHZThxC1nZbgWHwAAAJfSas6u+etJjkxyjQzH010/yYuS3HF77+nuk3LJxdNXeufqugkAAMCuWM2JVx6bYWTu/CTp7s8mufYUnQIAAGD3rCbkXdjd393yoKr2zbC7JQAAAHuI1YS8E6vqqUkuX1V3SvKGJG+fplsAAADsjtWcXfPJSR6R5OMZTpTyziQvmaJTMKV3vfRuk7V910c43BQAgMVazdk1L07y4vEGAADAHminIa+qXt/dD6yqj2cbx+B19y0n6RkAAACrtisjeUeNP+8xZUcAAAC49HYa8rr7nPHuZZKc093fSZKqunyS60zYNwAAAFZpNWfXfEOSi1c8vmicBgAAwB5iNSFv35XXyRvvX3btuwQAAMDuWs0lFDZX1b26+21JUlX3TvKNaboF8/L6lx82WdsP/JXjJ2sbAIC9z2pC3qOSvKqqXpCkknw5yS9P0isAAAB2y2quk/f5JLepqislqe6+YLpuAQAAsDt25Tp5D+nuf6iqJ2w1PUnS3X81Ud8AAABYpV0ZybvC+PPKU3YEAACAS29XQt6B488zutslEwAAAPZgu3IJhbtV1X5JnjJ1ZwAAALh0dmUk7/gMl0q4YlWdv2J6JenuvsokPQMAAGDVdmUk72ndfdUk7+juq6y4XVnAAwAA2LPsSsj78Pjz/B2+CgAAgIXbld01L1tVD0ty26q679ZPdveb1r5bAAAA7I5dCXmPSvLgJFdLcs+tnuskQh4AAMAeYqchr7tPSnJSVW3q7peuQ58AAADYTbtyTN4Wr62qp1XV0UlSVTepqntM1C8AAAB2w2pC3suSfDfJbcfHZyf5kzXvEQAAALttNSHvwO5+ZpLvJUl3/3eGa+UBAACwh1hNyPtuVV0+w8lWUlUHJrlwkl4BAACwW3bl7Jpb/EGS45P8YFW9Ksntkjx8ik4BAACwe3Y55HX3e6vqlCS3ybCb5lHd/Y3JegYAAMCqrWYkLxlG726/4vFxa9gXAAAALqVdPiavqp6R5KgkZ4y3o6rqz6fqGAAAAKu3mpG8uyU5qLsvTpKqOibJqUmeMkXHAAAAWL3VnF0zSa624v5V17IjAAAAXHqrGcn78ySnVtUJGU68cvsYxQMAANijrObsmq+pqvcnuXWGkPek7v7aVB0DAABg9VZz4pX7JPl2d7+tu9+a5DtVdfh0XQMAAGC1VnNM3h90939uedDd38xwgXQAAAD2EKsJedt67WqvswcAAMCEVhPyNlXVX1XVgVX1w1X1nCQnT9UxAAAAVm81Ie9xSb6b5HVJXp/kv5M8dopOAQAAsHtWc3bN/0ry5O09X1XP7+7HbTXtB5P8fZLrJrk4ydHd/dyqukaGsHhAkrOSPLC7z1t17wEAAPhfVnsx9B253TamfT/Jb3f3jya5TZLHVtXNM4TF93X3TZK8LzsIjwAAAOy6tQx5/0d3n9Pdp4z3L0hyZpLrJ7l3kmPGlx2TxKUYAAAA1sCkIW+lqjogycFJPpLkOt19TjIEwSTXXq9+AAAAzNlahrza7hNVV0pybJLHd/f5u9xg1ZFVtamqNm3evHkt+ggAADBruxXyquoyVXWVrSY/dzuv3S9DwHtVd79pnPz1qrre+Pz1kpy7rfd299HdvbG7N27YsGF3ugoAALBUdjnkVdWrq+oqVXXFJGck+XRVPXHL8939im28p5K8NMmZ3f1XK556W5KHjfcfluStu9F3AAAAtrKakbybj7taHp7knUl+KMlDd/Ke242vObSqThtvd0vyjCR3qqrPJrnT+BgAAIBLaZevk5dkv3HXy8OTvKC7v1dVvaM3dPdJ2f6xendcRW0AAAB2wWpG8v4uw4XLr5jkA1V1wyS7fBIVAAAAprfLI3nd/bwkz1sx6UtVdYe17xIAAAC7azUnXrlmVT2vqk6pqpOr6rlJrjph3wAAAFil1eyu+dokm5PcL8n9x/uvm6JTAAAA7J7VnHjlGt39xyse/0lVHb7WHQLWxkv+/i6Ttf1rv/zuydoGAODSWc1I3glVdcR4IfTLVNUDk7xjqo4BAACwejsdyauqC5J0hkshPCHJK8en9knyrSR/MFnvAAAAWJWdhrzuvvKW+1V1jSQ3SbL/lJ0CAABg9+zyMXlV9WtJjkpygySnJblNkg/FRc0BAAD2GKs5Ju+oJLdO8qXuvkOSg5N8Y5JeAQAAsFtWE/K+093fSZKqulx3fyrJTafpFgAAALtjNZdQOLuqrpbkLUneW1XnJfnqNN0CAABgd+xyyOvu+4x3n15VJyS5apLjJ+kVAAAAu2U1I3n/o7tPXOuOAAAAcOmt5pg8AAAA9nBCHgAAwIwIeQAAADMi5AEAAMyIkAcAADAjQh4AAMCMCHkAAAAzIuQBAADMiJAHAAAwI0IeAADAjAh5AAAAMyLkAQAAzIiQBwAAMCNCHgAAwIwIeQAAADMi5AEAAMyIkAcAADAjQh4AAMCMCHkAAAAzIuQBAADMiJAHAAAwI0IeAADAjAh5AAAAMyLkAQAAzIiQBwAAMCP7Ttl4Vb0syT2SnNvdtxinPT3JryfZPL7sqd39zin7AayPv3r1XSZr+wm/9O7J2gYAmJOpR/JekeSwbUx/TncfNN4EPAAAgDUyacjr7g8k+Y8pawAAAHCJRR2T9xtVdXpVvayqrr6gPgAAAMzOIkLeC5McmOSgJOck+cvtvbCqjqyqTVW1afPmzdt7GQAAAKN1D3nd/fXuvqi7L07y4iSH7OC1R3f3xu7euGHDhvXrJAAAwF5q3UNeVV1vxcP7JPnEevcBAABgrqa+hMJrkvx8kmtV1dlJ/iDJz1fVQUk6yVlJHjllHwAAAJbJpCGvux+0jckvnbImAADAMlvU2TUBAACYgJAHAAAwI0IeAADAjAh5AAAAMzLpiVcApvaUNxw2Wdt//oDjJ2sbAGAqRvIAAABmRMgDAACYESEPAABgRoQ8AACAGRHyAAAAZkTIAwAAmBEhDwAAYEaEPAAAgBkR8gAAAGZEyAMAAJgRIQ8AAGBGhDwAAIAZEfIAAABmRMgDAACYESEPAABgRoQ8AACAGRHyAAAAZkTIAwAAmBEhDwAAYEaEPAAAgBkR8gAAAGZEyAMAAJgRIQ8AAGBGhDwAAIAZEfIAAABmRMgDAACYESEPAABgRoQ8AACAGRHyAAAAZkTIAwAAmBEhDwAAYEaEPAAAgBkR8gAAAGZEyAMAAJgRIQ8AAGBGJg15VfWyqjq3qj6xYto1quq9VfXZ8efVp+wDAADAMpl6JO8VSQ7batqTk7yvu2+S5H3jYwAAANbApCGvuz+Q5D+2mnzvJMeM949JcviUfQAAAFgmizgm7zrdfU6SjD+vvYA+AAAAzNIefeKVqjqyqjZV1abNmzcvujsAAAB7vEWEvK9X1fWSZPx57vZe2N1Hd/fG7t64YcOGdesgAADA3mrfBdR8W5KHJXnG+POtC+gDwG65+9u2PpfU2nnHvY6frG0AYHlMfQmF1yT5cJKbVtXZVfWIDOHuTlX12SR3Gh8DAACwBiYdyevuB23nqTtOWRcAAGBZ7dEnXgEAAGB1hDwAAIAZEfIAAABmRMgDAACYESEPAABgRoQ8AACAGRHyAAAAZkTIAwAAmBEhDwAAYEaEPAAAgBkR8gAAAGZEyAMAAJgRIQ8AAGBGhDwAAIAZEfIAAABmRMgDAACYESEPAABgRoQ8AACAGRHyAAAAZmTfRXcAgB2721ueNFnb7zz8LyZrGwBYDCN5AAAAMyLkAQAAzIiQBwAAMCNCHgAAwIwIeQAAADMi5AEAAMyIkAcAADAjQh4AAMCMCHkAAAAzIuQBAADMiJAHAAAwI0IeAADAjAh5AAAAMyLkAQAAzIiQBwAAMCNCHgAAwIwIeQAAADOy76I7AMCe5+5vfuZkbb/jPr87WdsAgJE8AACAWRHyAAAAZmRhu2tW1VlJLkhyUZLvd/fGRfUFAABgLhZ9TN4duvsbC+4DAADAbNhdEwAAYEYWGfI6yXuq6uSqOnJbL6iqI6tqU1Vt2rx58zp3DwAAYO+zyJB3u+7+iSR3TfLYqrr91i/o7qO7e2N3b9ywYcP69xAAAGAvs7CQ191fHX+em+TNSQ5ZVF8AAADmYiEhr6quWFVX3nI/yZ2TfGIRfQEAAJiTRZ1d8zpJ3lxVW/rw6u4+fkF9AQAAmI2FhLzu/kKSWy2iNgAAwJwt+jp5AJAkufubXjhZ2++476MnaxsA9jSukwcAADAjQh4AAMCMCHkAAAAz4pg8AJbWPY49ZrK2j7vfwyZrGwB2xEgeAADAjAh5AAAAMyLkAQAAzIiQBwAAMCNCHgAAwIw4uyYArKN7vPF1k7V93P1/cbK2Adh7GMkDAACYESEPAABgRoQ8AACAGRHyAAAAZkTIAwAAmBEhDwAAYEaEPAAAgBkR8gAAAGZEyAMAAJgRIQ8AAGBGhDwAAIAZEfIAAABmRMgDAACYkX0X3QEAYFr3euPbJ2v7bfe/5/+Zdvix/zRZvbfc79DJ2gaYCyEPANjr3e/YTZO1fez9Nk7WNsAU7K4JAAAwI0byAAB2wxFvOmuytl973wMmaxuYPyEPAGAv8aw3f22ytp94n+tO1jawvuyuCQAAMCNG8gAA2K7XHvuNydo+4n7XmqxtWGZG8gAAAGZEyAMAAJgRu2sCALBHee9rNk/W9p0etGGytmFPIeQBALD0Nr3s3Mna3vir156sbdgWIQ8AANbZ55//9UnbP/Bx15m0ffZsQh4AACyBrz37i5O1fd3fudFkbbN6TrwCAAAwI0byAACASXz9OadO1vZ1fuvgydre2y1sJK+qDquqT1fV56rqyYvqBwAAwJwsZCSvqvZJ8jdJ7pTk7CT/WlVv6+4zFtEfAABgHr7+vBMna/s6v/lzk7W9lha1u+YhST7X3V9Ikqp6bZJ7JxHyAACAvcq5f3PcZG1f+7H3WPV7FrW75vWTfHnF47PHaQAAAFwK1d3rX7TqAUnu0t2/Nj5+aJJDuvtxW73uyCRHjg9vmuTTu1HuWkm+cSm6uzvUVHNvqqemmntjzWWYRzXV3BtrLsM8qqnmIuvdsLs37OxFi9pd8+wkP7ji8Q2SfHXrF3X30UmOvjSFqmpTd2+8NG2oqeZ61lyGeVRTzb2tnppqqrln1lNTzb2x5nrUW9Tumv+a5CZVdaOqumySI5K8bUFykwFiAAARGklEQVR9AQAAmI2FjOR19/er6jeSvDvJPkle1t2fXERfAAAA5mRhF0Pv7ncmeec6lLpUu3uqqeYCai7DPKqp5t5WT0011dwz66mp5t5Yc/J6CznxCgAAANNY1DF5AAAATEDIAwAAmBEhDwAAYEaEvIlU1Z0mbPu6VXXd8f6GqrpvVf3YVPWWRVX9UFXtP96vqvqVqnp+VT26qhZ2kqIpjZcxuW9V3WzRfZmLqnrfrkxbw3pXqqr7V9VvVdXjquqwqrJtvxSq6l5btgXLpqr+bNF9YPdV1VWq6sBtTL/lIvoDO1NVt6+qm473f6aqfqeq7r7ofs3BUnwRqKpFnKXnpVM0WlWPTPLhJP9SVY9OclySeyR5U1U9YoqaO+nPJMu2qvapqkdW1R9X1e22eu5pU9TMcLbXLb8Tz0hy9yQfSXLrTHQWpKr64ap6WVX9yfhl/cVV9YmqekNVHTBBvbesuH/vJP+U5J5J3lpVD1/remOdK1TV71bVE6tq/6p6eFW9raqeWVVXmqjmLVfc36+qnjbW/LOqusJENfevqmskuVZVXb2qrjHeDkjyAxPVfGCSE5IcluQ3khyS5KFJTquqH5+o5iI+zzdV1UOman8bXpfk7Kp6ZVXdrar2mbpgVe07bvOOr6rTq+pjVfWuqnpUVe03Uc3nbXV7fpLHbHk8Uc113ebtoB+fmbj99V5nt2wPPpXk2Kr6ZFXdesXTr5ig3m9U1bXG+zeuqg9U1Ter6iMTbn8uU1W/WlXvGH9HTq6q11bVz09Rb6y5iO8j2+rH1OvsIv5u/nWG71uvrKo/TvLMJJdP8ltV9ayJai5ivV337Xsyo7Nrjl+utvlUko919w0mqLm9C7hXkkO7+4oT1Px4kp/K8EvwpSQ37u6vVdXVk5zQ3QdNUHMRy/YlSa6Q5KMZvrSe2N1PGJ87pbt/YoKaZ3T3zcf7Jye5dXdfPD7+WHffaoKaH0jymiRXTfKQJC9P8vokd07y4O4+dI3rndrdB4/3PzTW+OK4wXvfRPP4+iRfzrDO3jTJmRnm8Z5JrtvdD52g5v+sI1X1l0mumWHZHp7kmt39yxPUPCrJ4zMEuq9k+P1IkvOTvLi7XzBBzdOT3Ka7vz1+hq/q7ruMf6xf1N23naDmIj7Pr2T459ahSf4xw+/MO7r7u2tda6x36ljr/kmOSHKLJG9O8pruPnGimq9J8s0kxyQ5e5x8gyQPS3KN7v7FCWqeneT9Sd6TS9bXZyf5nSTp7mMmqLmu27yx5gVJtnzZ2TKfV0jy7STd3VeZoOa6rrNjzdOS3LW7z6mqQ5L8fZKndvebVm7717DeJ7v7x8b770jyku5+8xi4/rS7b7fDBnav5sszfPf5xwy/n+cn+WCSJyV5a3c/f4Kai/g+soh1dhF/Nz+ZYft6+Qx/N68//j3bL8mp3X2LKWouYL1d9+17kqS7Z3FLclGSLyT54orblsffnajmeRlGe35uq9vPJ/n6RDVPXXH/Y9t7bgbL9vQV9/fNMJL2piSXm3A+350hnCfJsUluON6/5tbLeqLP89+m/jyTnLLi/kfXaf05bfxZSb6WS/65VCs/5wmX62lJ9pu65op6j5uy/a1qfXzF8rz8VvP9ibl9nkmunOFL1juTbM7wBeTOE9Q7ZavH103ymxm+tH95onn89A6e+8xENa+S5K+TvDrDl6sk+cIUtbb+LMf7k2/zxnafnyHwXGfFtC+ux3yu1zo71vrEVo+vl+Tkcd09ZYJ6n15x/1+3em6qbcHpWz3+l/Hn5ZKcOXXNrN/3kYWts+P9dfm7uWWdTbJ/hu/Ulx8f75PkjIlqLmK9Xffte3cv7mLoE/hCkjt2979t/URVfXmimv+S5Nu9jf/sVtWnJ6p5UVXt193fyxAwt9TbP9PtfruIZXvZLXe6+/tJjqyq38+we+FUu7/8WpK/r6qnJ/nPDLu8nZrk6kmeMFHNi6vqRzL8V/sKVbWxuzdV1Y0zbOTW2i2r6vwMG+39q+q6PYwEX3aiev+ju7uq3tnjVm18PNWuBFetqvtk+J243Pj7MnXNjDWeX1W3TXJAcsk2trv/foJy70hyfFWdmOSuSd6Q/M/oe+3ojZfWOn+eW2pckOSVGXbtuUaSByZ5coaRqMl099eSPC/J86rqhhOVOa+qHpDk2L5kD4LLJHlAhi8/a667z0/y+Kr6yST/MP5Xe+rDONZ7m5fuftw4j6+pYZf1F+SSUZKpLGKdPb+qDuzuz4+1zxlHJ96SZIrj9t9YVa9I8kdJ3lxVj88Qfu6Y5P98X1gj39syj1X1E0m+myTdfeGE2591/z6yoHV2EX8331FVJ2UIzC9J8vqq+pcMAyYfmKjmItbbdd++J5lVyPvrDF/Gt/UBPXOiml/IuIHZWnfffqKap2fYXfOk7j57xfRrJvntiWouYtluqqrDuvv4LRO6+4+q6qtJXjhRzScleVqGX7ibZDiG4ewM/+m5eKKav5vk7UkuzrBLxFOq6lYZ/sP+6xPU+7skr+7uf95q+hWSPHKCesnwWV6pu7/V3b+6ZWINJwe4YKKaJya513j/X6rqOt399RpOWPSNiWomSarqlUkOzPCf0IvGyZ3hv7Jr7cpJTkpyYZI/7O5/HKd/M8ma70I0WsTn+a2tJ3T3fyR50Xhba/tX1W27+0PbqPulCeolw26hf5Hkb6tqyx/9q2U45vKIKQpW1QsybA8+VFWHJnlMhvVpSuu9zUuSdPfJVfULGY5dPTHDyMGU1nudTYa/XT+Q5PMral5QVYdlCJdrqrv/Xw3Hcr8mwzbvckmOzBAqH7zW9UZPTHJCVX0nyX4ZfzeqakOGcxRMYRHfRxaxzi7i7+aVkjwlw15hHxn/jtwnQ+B74xQFF7Tervv2PZnRMXmLMB6Dc0SGXSJel+F4jdPmVnNZ7CnLdjy26rzuvminL15923vEPK7oT/XMNkJVdWaSm6/HfK34PH8gyWvj81wTi/49qaprZvj7PPU/JPaI7cGU27zt1LtekoO7+53rUW+97Cmf59SqqjIcIzbp78eexDo7H+u1fU+WJORV1Z26+70Ttn/DDCvpERn+0/KaJK/t7snOhLTeNavqKkk2bNkNZMX0W3b36XOpOba/DMt29vO4g75MvT14Q5Lf7O5zpqqxjZrrvg3aTj8mXbbrXXNZluuyzOd619zD/oa9prs/O0Gtpdm2r3dN37vmtQ1ayLLtiQ7225Nu2erg7olrHZzk1CQXzaVmht08vpph97NPZjjr5Jbn1vxg7kXVXJZlu4zzuFV/Jt0eZNj94rwMJ/J525bbOs7fum+D1mvZLrLmsizXZZnPqWvuKdu9KT/PPWUerT97b83t9GN226BFLdvZHJNXO76cwTUnrr1fhutUHZHhwM0Tk/zhjGo+NclP9iWnZX5lVT21u9+U6U7usIiaSZZi2c5+Hhe5PUjy9Inb/z/W8/NcxLJd1Oc59+W6ovas53NBy3YZ/oYtxbZ9idafZVhnl+nznE/IS/KzGa65s/XBzpXhAsFrrqrulORBGc5y+dEMx8Qc2d3/NUW9RdVMsm+Pu55190er6g5JjquqG2S6sz2te81lWLbLMI+jdd8ebNETXUdtWxb0eS5i2a5rzWVZrssynwuquQx/w5Zl274U688iai7RNmgRn+d8dtdM8q4kd9jOcx+YqOYJGc4Gdo11nM9F1PxQkgO3mnblJO9LcuGMas5+2S7DPI7tr/v2YEX7F2S4QO/5Sb6T4Qyb509UaxGf5yK2tetac4mW67LM5yJqzv5v2LJs25do/Zn9OrtMn2f3jHbXzAIuZ9Ddd5ii3T2tZtb5tMyLqrkMy3YZ5nG0iMubbGn/yisfV9Xhmei/gwv6PBexbNe15rIs12WZzwXVXIa/YcuybV+K9WcRNZdoG7SIz3Pyi56up88keXZVnVVVf1FVBy26QzPyniTP3HrZdvf3uvtVM6q5CMswn4uYxz1me9Ddb0ly6KLqT2ARy3aP+TwntAzzmCzP+mPbPg3rj+9dl9ayfJ7zu4RC7SGnYZ2j7SzbSU7LvMiai7AM87kHrT9TX97kviseXibJxiQ/190/PVXNRVjQsp399n0Z5jFZnvXHtn1da1p/9tKai7AMn+fsQt5KVXVwkpcluWV377Po/szJIpbtsnyeyzCfc15/qurlKx5+P8lZSV7c3edOVXPR5vx5LtIyzGOyPOvPMnyey7Jc1ZyXuS7bOe2umWQ4DWtV3bOqXpXh4MrPJLnfgrs1C4tYtsvyeS7DfC7L+tPdv7Li9uvd/adzDHjL8nmut2WYx2R51p9l+DyXZbmqOS/LsGxnM5JX2z4N61t62tOwLoVFLNtl+TyXYT6Xbf2p4ZTIz09yuwynRj4pyVHdffbUtdfDsn2e62UZ5jFZnvVnGT7PZVmuas5nnU2Wa9nOKeSdkOTVSY7t7v9YdH/mZBHLdlk+z2WYz2Vbf6rqvWPtV46THpLkwd19p/Xsx1SW7fNcL8swj8nyrD/L8Hkuy3JVc16WadnOJuQB7Amq6rTuPmhn0wAApjK7Y/IAFuwbVfWQqtpnvD0kyb8vulMAwPIwkgewhqrqh5K8IMlPZzgm70MZjsn70kI7BgAsDSEPAABgRvZddAcA5qSqbpTkcUkOyIptbHffa1F9AgCWi5AHsLbekuSlSd6e5OIF9wUAWEJ21wRYQ1X1ke7+qUX3AwBYXkIewBqqql9KcpMk70ly4Zbp3X3KwjoFACwVu2sCrK0fT/LQJIfmkt01e3wMADA5I3kAa6iqPpXklt393UX3BQBYTi6GDrC2PpbkaovuBACwvOyuCbC2rpPkU1X1r7nkmLzu7nsvsE8AwBKxuybAGqqqn1v5MMnPJHlQd//YgroEACwZu2sCrKHuPjHJfya5e5JXJLljkhctsk8AwHKxuybAGqiqH0lyRJIHJfn3JK/LsLfEHRbaMQBg6dhdE2ANVNXFST6Y5BHd/blx2he6+4cX2zMAYNnYXRNgbdwvydeSnFBVL66qO2Y4Jg8AYF0ZyQNYQ1V1xSSHZ9ht89AkxyR5c3e/Z6EdAwCWhpAHMJGqukaSByT5xe4+dNH9AQCWg5AHAAAwI47JAwAAmBEhDwAAYEaEPABmraouqqrTVtwO2I02rlZVj1n73gHA2nNMHgCzVlXf6u4rXco2DkhyXHffYpXv26e7L7o0tQFgtYzkAbB0qmqfqnpWVf1rVZ1eVY8cp1+pqt5XVadU1cer6t7jW56R5MBxJPBZVfXzVXXcivZeUFUPH++fVVW/X1UnJXlAVR1YVcdX1clV9cGqutl6zy8Ay2XfRXcAACZ2+ao6bbz/xe6+T5JHJPnP7r51VV0uyT9X1XuSfDnJfbr7/Kq6VpJ/qaq3JXlyklt090FJUlU/v5Oa3+nunxlf+74kj+ruz1bVTyX52wzXUASASQh5AMzdf28JZyvcOcktq+r+4+OrJrlJkrOT/FlV3T7JxUmun+Q6u1HzdckwMpjktkneUFVbnrvcbrQHALtMyANgGVWSx3X3u//XxGGXyw1JfrK7v1dVZyXZfxvv/37+9yEPW7/mv8afl0nyzW2ETACYjGPyAFhG707y6KraL0mq6keq6ooZRvTOHQPeHZLccHz9BUmuvOL9X0py86q6XFVdNckdt1Wku89P8sWqesBYp6rqVtPMEgAMhDwAltFLkpyR5JSq+kSSv8uwd8urkmysqk1JHpzkU0nS3f+e4bi9T1TVs7r7y0len+T08T2n7qDWg5M8oqo+luSTSe69g9cCwKXmEgoAAAAzYiQPAABgRoQ8AACAGRHyAAAAZkTIAwAAmBEhDwAAYEaEPAAAgBkR8gAAAGZEyAMAAJiR/w8PCE+RDWvvzAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision is  0.8227848101265823\n",
      "Recall is  0.7142857142857143\n",
      "F1 score is  0.7647058823529411\n"
     ]
    }
   ],
   "source": [
    "#feature coefficient\n",
    "features = ['V1','V2','V3','V4','V5','V6','V7','V8','V9',\n",
    "                                            'V10','V11','V12','V13','V14','V15','V16','V17','V18',\n",
    "                                            'V19','V20','V21','V22','V23','V24','V25','V26','V27',\n",
    "                                            'V28','Amount']\n",
    "lr = LogisticRegression(featuresCol = 'features', labelCol='Class',threshold=0.2, maxIter=150)\n",
    "lrModel = lr.fit(df_train)\n",
    "coefficients = lrModel.coefficients\n",
    "abs_coefficients=[abs(i) for i in coefficients]\n",
    "tmp = pd.DataFrame({'Feature': features, 'abs_coefficients': abs_coefficients})\n",
    "tmp = tmp.sort_values(by='abs_coefficients',ascending=False)\n",
    "plt.figure(figsize = (15,8))\n",
    "plt.title('abs_coefficients',fontsize=14)\n",
    "s = sns.barplot(x='Feature',y='abs_coefficients',data=tmp)\n",
    "s.set_xticklabels(s.get_xticklabels(),rotation=90)\n",
    "plt.show()\n",
    "\n",
    "pred = lrModel.transform(df_test)\n",
    "pred = pred.select(\"features\", \"Class\", \"rawPrediction\", \"prediction\")\n",
    "precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.prediction == 1.0).count()\n",
    "recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.Class == 1.0).count()\n",
    "f1 = 2 * precision * recall /(precision + recall)\n",
    "print(\"Precision is \", precision)\n",
    "print(\"Recall is \", recall)\n",
    "print(\"F1 score is \", f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step5. stratified cv using features selected from 4)\n",
    "#### 'V11', 'V1','V21','V3','V22','V6','V28'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.2, 150]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "\n",
      " precision, recall, f_score: 0.8205128205128205, 0.5245901639344263, 0.64\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.7608695652173914, 0.5645161290322581, 0.6481481481481481\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.9024390243902439, 0.5967741935483871, 0.7184466019417476\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.84375, 0.43548387096774194, 0.5744680851063829\n",
      "\n",
      " precision, recall, f_score: 0.8918918918918919, 0.5409836065573771, 0.673469387755102\n",
      "grid search precision: 0.8438926604024696\n",
      "grid search recall: 0.5324695928080381\n",
      "grid search f-score: 0.6529490041012765\n",
      "grid search hyper: [0.2, 150]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "df = sqlContext.read.csv(\"base_train.csv\", header = True)\n",
    "features = ['V11', 'V1','V21','V3','V22','V6','V28']\n",
    "gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None',0, False, features, 'lr', [0.2],[150])\n",
    "print(\"grid search precision:\", gs_precision)\n",
    "print(\"grid search recall:\", gs_recall)\n",
    "print(\"grid search f-score:\", gs_fscore)\n",
    "print(\"grid search hyper:\", gs_hyper)\n",
    "sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step6. revisit step5 to choose best feature"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 'V11', 'V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.2, 150]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "\n",
      " precision, recall, f_score: 0.9090909090909091, 0.6557377049180327, 0.761904761904762\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8775510204081632, 0.6935483870967742, 0.7747747747747746\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8, 0.6451612903225806, 0.7142857142857142\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8260869565217391, 0.6129032258064516, 0.7037037037037037\n",
      "\n",
      " precision, recall, f_score: 0.8333333333333334, 0.5737704918032787, 0.6796116504854368\n",
      "grid search precision: 0.849212443870829\n",
      "grid search recall: 0.6362242199894236\n",
      "grid search f-score: 0.7274487534230674\n",
      "grid search hyper: [0.2, 150]\n"
     ]
    }
   ],
   "source": [
    "df = sqlContext.read.csv(\"base_train.csv\", header = True)\n",
    "features = ['V11', 'V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10']\n",
    "gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None',0, False, features, 'lr', [0.2],[150])\n",
    "print(\"grid search precision:\", gs_precision)\n",
    "print(\"grid search recall:\", gs_recall)\n",
    "print(\"grid search f-score:\", gs_fscore)\n",
    "print(\"grid search hyper:\", gs_hyper)\n",
    "sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 'V11','V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10','V4','V18','V14','V27','V2','V26','V19','V12','V24','V25','V8','V16'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.2, 150]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.7818181818181819, 0.6935483870967742, 0.7350427350427351\n",
      "\n",
      " precision, recall, f_score: 0.7307692307692307, 0.6229508196721312, 0.672566371681416\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.7352941176470589, 0.8064516129032258, 0.7692307692307693\n",
      "\n",
      " precision, recall, f_score: 0.8958333333333334, 0.6935483870967742, 0.7818181818181819\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8372093023255814, 0.5901639344262295, 0.6923076923076923\n",
      "grid search precision: 0.7961848331786773\n",
      "grid search recall: 0.681332628239027\n",
      "grid search f-score: 0.7342948142665915\n"
     ]
    }
   ],
   "source": [
    "df = sqlContext.read.csv(\"base_train.csv\", header = True)\n",
    "features = ['V11', 'V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10','V4','V18','V14','V27','V2','V26','V19','V12','V24','V25','V8','V16']\n",
    "gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None',0, False, features, 'lr', [0.2],[150])\n",
    "print(\"grid search precision:\", gs_precision)\n",
    "print(\"grid search recall:\", gs_recall)\n",
    "print(\"grid search f-score:\", gs_fscore)\n",
    "sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step7. Using finalized features do stratified cv grid search for marginal improvement\n",
    "#### finalized feature from step6: 'V11','V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10','V4','V18','V14','V27','V2','V26','V19','V12','V24','V25','V8','V16'\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.1, 130]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "\n",
      " precision, recall, f_score: 0.7352941176470589, 0.819672131147541, 0.7751937984496124\n",
      "\n",
      " precision, recall, f_score: 0.7540983606557377, 0.7419354838709677, 0.7479674796747967\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.7166666666666667, 0.6935483870967742, 0.7049180327868854\n",
      "\n",
      " precision, recall, f_score: 0.8148148148148148, 0.7096774193548387, 0.7586206896551724\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.84, 0.6885245901639344, 0.7567567567567568\n",
      "[0.1, 150]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "\n",
      " precision, recall, f_score: 0.7540983606557377, 0.7419354838709677, 0.7479674796747967\n",
      "\n",
      " precision, recall, f_score: 0.7575757575757576, 0.819672131147541, 0.7874015748031497\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.7166666666666667, 0.6935483870967742, 0.7049180327868854\n",
      "\n",
      " precision, recall, f_score: 0.8148148148148148, 0.7096774193548387, 0.7586206896551724\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.84, 0.6885245901639344, 0.7567567567567568\n",
      "[0.1, 170]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.7575757575757576, 0.819672131147541, 0.7874015748031497\n",
      "\n",
      " precision, recall, f_score: 0.7540983606557377, 0.7419354838709677, 0.7479674796747967\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.7166666666666667, 0.6935483870967742, 0.7049180327868854\n",
      "\n",
      " precision, recall, f_score: 0.8148148148148148, 0.7096774193548387, 0.7586206896551724\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8235294117647058, 0.6885245901639344, 0.7499999999999999\n",
      "[0.15, 130]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.7368421052631579, 0.6774193548387096, 0.7058823529411764\n",
      "\n",
      " precision, recall, f_score: 0.7619047619047619, 0.7868852459016393, 0.7741935483870968\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8627450980392157, 0.7096774193548387, 0.7787610619469026\n",
      "\n",
      " precision, recall, f_score: 0.7368421052631579, 0.6774193548387096, 0.7058823529411764\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8367346938775511, 0.6721311475409836, 0.7454545454545455\n",
      "[0.15, 150]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "\n",
      " precision, recall, f_score: 0.7413793103448276, 0.6935483870967742, 0.7166666666666668\n",
      "\n",
      " precision, recall, f_score: 0.7619047619047619, 0.7868852459016393, 0.7741935483870968\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.7636363636363637, 0.6774193548387096, 0.7179487179487181\n",
      "\n",
      " precision, recall, f_score: 0.8627450980392157, 0.7096774193548387, 0.7787610619469026\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8333333333333334, 0.6557377049180327, 0.7339449541284403\n",
      "[0.15, 170]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.7413793103448276, 0.6935483870967742, 0.7166666666666668\n",
      "\n",
      " precision, recall, f_score: 0.7619047619047619, 0.7868852459016393, 0.7741935483870968\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8461538461538461, 0.7096774193548387, 0.7719298245614036\n",
      "\n",
      " precision, recall, f_score: 0.7636363636363637, 0.6774193548387096, 0.7179487179487181\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8367346938775511, 0.6721311475409836, 0.7454545454545455\n",
      "[0.2, 130]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "\n",
      " precision, recall, f_score: 0.7833333333333333, 0.7704918032786885, 0.7768595041322314\n",
      "\n",
      " precision, recall, f_score: 0.7407407407407407, 0.6451612903225806, 0.689655172413793\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.7777777777777778, 0.6774193548387096, 0.7241379310344828\n",
      "\n",
      " precision, recall, f_score: 0.8723404255319149, 0.6612903225806451, 0.7522935779816514\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8297872340425532, 0.639344262295082, 0.7222222222222223\n",
      "[0.2, 150]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "\n",
      " precision, recall, f_score: 0.7833333333333333, 0.7704918032786885, 0.7768595041322314\n",
      "\n",
      " precision, recall, f_score: 0.7407407407407407, 0.6451612903225806, 0.689655172413793\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8723404255319149, 0.6612903225806451, 0.7522935779816514\n",
      "\n",
      " precision, recall, f_score: 0.7777777777777778, 0.6774193548387096, 0.7241379310344828\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8333333333333334, 0.6557377049180327, 0.7339449541284403\n",
      "[0.2, 170]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "\n",
      " precision, recall, f_score: 0.7407407407407407, 0.6451612903225806, 0.689655172413793\n",
      "\n",
      " precision, recall, f_score: 0.7833333333333333, 0.7704918032786885, 0.7768595041322314\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "\n",
      " precision, recall, f_score: 0.8541666666666666, 0.6612903225806451, 0.7454545454545455\n",
      "\n",
      " precision, recall, f_score: 0.7777777777777778, 0.6774193548387096, 0.7241379310344828\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "\n",
      " precision, recall, f_score: 0.8333333333333334, 0.6557377049180327, 0.7339449541284403\n",
      "grid search precision: 0.7766311199425953\n",
      "grid search recall: 0.7306716023268112\n",
      "grid search f-score: 0.7529506799681837\n",
      "grid search hyper: [0.1, 150]\n"
     ]
    }
   ],
   "source": [
    "df = sqlContext.read.csv(\"base_train.csv\", header = True)\n",
    "features = ['V11', 'V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10','V4','V18','V14','V27','V2','V26','V19','V12','V24','V25','V8','V16']\n",
    "gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None', 0, False, features, 'lr', [0.1,0.15,0.2],[130,150,170])\n",
    "print(\"grid search precision:\", gs_precision)\n",
    "print(\"grid search recall:\", gs_recall)\n",
    "print(\"grid search f-score:\", gs_fscore)\n",
    "print(\"grid search hyper:\", gs_hyper)\n",
    "sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Best hyper parameter after second grid search\n",
    "#### grid search hyper: [0.1, 150]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step8\n",
    "## use stratified cv to compare no sampling, undersampling, and oversampling\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.1, 150]\n",
      "train: 24700\n",
      "fraud ratio: 0.01\n",
      "train: 24600\n",
      "fraud ratio: 0.01\n",
      "\n",
      " precision, recall, f_score: 0.5714285714285714, 0.8524590163934426, 0.6842105263157895\n",
      "\n",
      " precision, recall, f_score: 0.5434782608695652, 0.8064516129032258, 0.6493506493506493\n",
      "train: 24600\n",
      "fraud ratio: 0.01\n",
      "train: 24600\n",
      "fraud ratio: 0.01\n",
      "\n",
      " precision, recall, f_score: 0.6235294117647059, 0.8548387096774194, 0.7210884353741497\n",
      "\n",
      " precision, recall, f_score: 0.4049586776859504, 0.7903225806451613, 0.53551912568306\n",
      "train: 24700\n",
      "fraud ratio: 0.01\n",
      "\n",
      " precision, recall, f_score: 0.5531914893617021, 0.8524590163934426, 0.6709677419354838\n",
      "[0.1, 150]\n",
      "train: 4919\n",
      "fraud ratio: 0.05001016466761537\n",
      "train: 4939\n",
      "fraud ratio: 0.05001012350678275\n",
      "\n",
      " precision, recall, f_score: 0.10707070707070707, 0.8548387096774194, 0.19030520646319568\n",
      "\n",
      " precision, recall, f_score: 0.09302325581395349, 0.9180327868852459, 0.1689291101055807\n",
      "train: 4919\n",
      "fraud ratio: 0.05001016466761537\n",
      "train: 4919\n",
      "fraud ratio: 0.05001016466761537\n",
      "\n",
      " precision, recall, f_score: 0.07726597325408618, 0.8387096774193549, 0.1414965986394558\n",
      "\n",
      " precision, recall, f_score: 0.08917197452229299, 0.9032258064516129, 0.16231884057971013\n",
      "train: 4939\n",
      "fraud ratio: 0.05001012350678275\n",
      "\n",
      " precision, recall, f_score: 0.10855949895615867, 0.8524590163934426, 0.1925925925925926\n",
      "[0.1, 150]\n",
      "train: 2460\n",
      "fraud ratio: 0.1\n",
      "train: 2470\n",
      "fraud ratio: 0.1\n",
      "\n",
      " precision, recall, f_score: 0.04189723320158103, 0.8548387096774194, 0.07987942727957799\n",
      "\n",
      " precision, recall, f_score: 0.040249826509368494, 0.9508196721311475, 0.07723035952063916\n",
      "train: 2460\n",
      "fraud ratio: 0.1\n",
      "train: 2460\n",
      "fraud ratio: 0.1\n",
      "\n",
      " precision, recall, f_score: 0.041514930808448654, 0.9193548387096774, 0.0794425087108014\n",
      "\n",
      " precision, recall, f_score: 0.04937163375224417, 0.8870967741935484, 0.0935374149659864\n",
      "train: 2470\n",
      "fraud ratio: 0.1\n",
      "\n",
      " precision, recall, f_score: 0.0425531914893617, 0.9180327868852459, 0.08133623819898331\n",
      "[0.1, 150]\n",
      "train: 1230\n",
      "fraud ratio: 0.2\n",
      "train: 1235\n",
      "fraud ratio: 0.2\n",
      "\n",
      " precision, recall, f_score: 0.02457831325301205, 0.8360655737704918, 0.047752808988764044\n",
      "\n",
      " precision, recall, f_score: 0.021574973031283712, 0.967741935483871, 0.042208934224410834\n",
      "train: 1230\n",
      "fraud ratio: 0.2\n",
      "train: 1230\n",
      "fraud ratio: 0.2\n",
      "\n",
      " precision, recall, f_score: 0.01762114537444934, 0.967741935483871, 0.034612056533025674\n",
      "\n",
      " precision, recall, f_score: 0.01858864027538726, 0.8709677419354839, 0.03640040444893832\n",
      "train: 1235\n",
      "fraud ratio: 0.2\n",
      "\n",
      " precision, recall, f_score: 0.01864406779661017, 0.9016393442622951, 0.036532713384257724\n",
      "grid search precision: 0.020201427946148508\n",
      "grid search recall: 0.9088313061872026\n",
      "grid search f-score: 0.039524312702009706\n",
      "grid search hyper: [0.1, 150]\n"
     ]
    }
   ],
   "source": [
    "#undersampling with finalized features and best param from step7:[0.1,150]\n",
    "df = sqlContext.read.csv(\"base_train.csv\", header = True)\n",
    "features = ['V11', 'V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10','V4','V18','V14','V27','V2','V26','V19','V12','V24','V25','V8','V16']\n",
    "for i in [0.01, 0.05, 0.10, 0.20]:\n",
    "    gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'under', i, False, features, 'lr', [0.1],[150])\n",
    "print(\"grid search precision:\", gs_precision)\n",
    "print(\"grid search recall:\", gs_recall)\n",
    "print(\"grid search f-score:\", gs_fscore)\n",
    "print(\"grid search hyper:\", gs_hyper)\n",
    "sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The results of oversampling and undersampling are not better than no sampling method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step9. Train base_train test base_test using hyperparams from 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision is  0.782608695652174\n",
      "Recall is  0.7912087912087912\n",
      "F1 score is  0.7868852459016393\n"
     ]
    }
   ],
   "source": [
    "#base model after feature selection\n",
    "\n",
    "trainData = sqlContext.read.csv(\"base_train.csv\", header = True)\n",
    "testData = sqlContext.read.csv(\"base_test.csv\", header = True)\n",
    "trainData = trainData.select(*(F.col(c).cast(\"double\").alias(c) for c in trainData.columns))\n",
    "testData = testData.select(*(F.col(c).cast(\"double\").alias(c) for c in testData.columns))\n",
    "train_count = trainData.select(\"Class\").count()\n",
    "train_fraud_count = trainData.select(\"Class\").where(trainData.Class == 1).count()\n",
    "test_count = testData.select(\"Class\").count()\n",
    "features = ['V11', 'V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10','V4','V18','V14','V27','V2','V26','V19','V12','V24','V25','V8','V16']\n",
    "vectorAssembler = VectorAssembler(inputCols = features, outputCol = 'features')\n",
    "df_train = vectorAssembler.transform(trainData)\n",
    "df_test = vectorAssembler.transform(testData)\n",
    "\n",
    "lr = LogisticRegression(featuresCol = 'features', labelCol='Class', threshold=0.1, maxIter=150 )\n",
    "lrModel = lr.fit(df_train)\n",
    "\n",
    "\n",
    "pred = lrModel.transform(df_test)\n",
    "pred = pred.select(\"features\", \"Class\", \"rawPrediction\", \"prediction\")\n",
    "precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.prediction == 1.0).count()\n",
    "recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.Class == 1.0).count()\n",
    "f1 = 2 * precision * recall /(precision + recall)\n",
    "print(\"Precision is \", precision)\n",
    "print(\"Recall is \", recall)\n",
    "print(\"F1 score is \", f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### result after second grid search with final best hyper param\n",
    "#### Precision is  0.782608695652174\n",
    "#### Recall is  0.7912087912087912\n",
    "#### F1 score is  0.7868852459016393\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step10. train base_train test oot using hyperparams from 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision is  0.7285714285714285\n",
      "Recall is  0.6891891891891891\n",
      "F1 score is  0.7083333333333334\n"
     ]
    }
   ],
   "source": [
    "#base model after feature selection\n",
    "\n",
    "trainData = sqlContext.read.csv(\"base_train.csv\", header = True)\n",
    "testData_oot = sqlContext.read.csv(\"oot.csv\", header = True)\n",
    "trainData = trainData.select(*(F.col(c).cast(\"double\").alias(c) for c in trainData.columns))\n",
    "testData_oot = testData_oot.select(*(F.col(c).cast(\"double\").alias(c) for c in testData_oot.columns))\n",
    "train_count = trainData.select(\"Class\").count()\n",
    "train_fraud_count = trainData.select(\"Class\").where(trainData.Class == 1).count()\n",
    "test_count_oot = testData_oot.select(\"Class\").count()\n",
    "features = ['V11', 'V1','V21','V3','V22','V6','V28','V5','V23','V15','Amount','V9','V13','V17','V7','V10','V4','V18','V14','V27','V2','V26','V19','V12','V24','V25','V8','V16']\n",
    "vectorAssembler = VectorAssembler(inputCols = features, outputCol = 'features')\n",
    "df_train = vectorAssembler.transform(trainData)\n",
    "df_test_oot = vectorAssembler.transform(testData_oot)\n",
    "\n",
    "lr = LogisticRegression(featuresCol = 'features', labelCol='Class', threshold=0.1, maxIter=150 )\n",
    "lrModel = lr.fit(df_train)\n",
    "\n",
    "\n",
    "pred = lrModel.transform(df_test_oot)\n",
    "pred = pred.select(\"features\", \"Class\", \"rawPrediction\", \"prediction\")\n",
    "precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.prediction == 1.0).count()\n",
    "recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.Class == 1.0).count()\n",
    "f1 = 2 * precision * recall /(precision + recall)\n",
    "print(\"Precision is \", precision)\n",
    "print(\"Recall is \", recall)\n",
    "print(\"F1 score is \", f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### extrapolation result\n",
    "#### Precision is  0.7285714285714285\n",
    "#### Recall is  0.6891891891891891\n",
    "#### F1 score is  0.7083333333333334"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}