text_featureTFIDF.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Method\n",
    "tf-idf + SVM(rbf\\linear)\n",
    "  \n",
    "  tf-idf + logistic(C\\...) ok\n",
    "  \n",
    "  tf-idf + MLP(ceng) ok\n",
    "\n",
    "------------------\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics import precision_score, recall_score, f1_score, roc_auc_score, confusion_matrix\n",
    "\n",
    "from sklearn import datasets\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.svm import SVC\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_train = np.load('words_dic.npy',allow_pickle=True).item()\n",
    "words_test = np.load('test_words_dic.npy',allow_pickle=True).item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = []\n",
    "for key in words_train:\n",
    "    sentance = ' '.join(words_train[key])\n",
    "    x_train.append(sentance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test = []\n",
    "for key in words_test:\n",
    "    sentance = ' '.join(words_test[key])\n",
    "    x_test.append(sentance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24500"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(24500, 26692)\n"
     ]
    }
   ],
   "source": [
    "# using tf-idf to represent words\n",
    "# for training data\n",
    "vectorizer = TfidfVectorizer(max_df=0.8, min_df=5)\n",
    "X_train = vectorizer.fit_transform(x_train)\n",
    "print(X_train.shape)\n",
    "X_train_a = X_train.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for testing data, using words from training\n",
    "tv2 = TfidfVectorizer(vocabulary = vectorizer.vocabulary_);\n",
    "X_test = tv2.fit_transform(x_test)\n",
    "X_test_a = X_test.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'has': 10729,\n",
       " 'be': 1902,\n",
       " 'funniest': 9521,\n",
       " 'stand': 22454,\n",
       " 'up': 25199,\n",
       " 'comedy': 4481,\n",
       " 'have': 10772,\n",
       " 'ever': 8097,\n",
       " 'seen': 20876,\n",
       " 'eddie': 7409,\n",
       " 'izzard': 12558,\n",
       " 'genius': 9758,\n",
       " 'he': 10806,\n",
       " 'picks': 17517,\n",
       " 'brits': 2890,\n",
       " 'americans': 743,\n",
       " 'everyone': 8106,\n",
       " 'between': 2170,\n",
       " 'his': 11121,\n",
       " 'style': 22895,\n",
       " 'completely': 4617,\n",
       " 'natural': 15792,\n",
       " 'hilarious': 11073,\n",
       " 'doubt': 6977,\n",
       " 'anyone': 977,\n",
       " 'could': 5174,\n",
       " 'sit': 21552,\n",
       " 'through': 23935,\n",
       " 'not': 16140,\n",
       " 'laugh': 13427,\n",
       " 'their': 23805,\n",
       " 'off': 16360,\n",
       " 'watch': 25868,\n",
       " 'enjoy': 7795,\n",
       " 'funny': 9523,\n",
       " 'noll': 16081,\n",
       " 'comfortable': 4489,\n",
       " 'way': 25905,\n",
       " 'rolling': 20086,\n",
       " 'out': 16642,\n",
       " 'blunt': 2453,\n",
       " 'comments': 4523,\n",
       " 'often': 16394,\n",
       " 'with': 26277,\n",
       " 'expletives': 8307,\n",
       " 'describe': 6203,\n",
       " 'things': 23858,\n",
       " 'more': 15402,\n",
       " 'knowledgeable': 13176,\n",
       " 'about': 52,\n",
       " 'than': 23782,\n",
       " 'most': 15455,\n",
       " 'quite': 18750,\n",
       " 'refreshing': 19253,\n",
       " 'there': 23829,\n",
       " 'one': 16458,\n",
       " 'other': 16627,\n",
       " 'character': 3736,\n",
       " 'film': 8819,\n",
       " 'constantly': 4893,\n",
       " 'tries': 24477,\n",
       " 'complicated': 4628,\n",
       " 'issues': 12539,\n",
       " 'using': 25270,\n",
       " 'language': 13364,\n",
       " 'necessary': 15841,\n",
       " 'guy': 10462,\n",
       " 'should': 21315,\n",
       " 'never': 15936,\n",
       " 'been': 1973,\n",
       " 'given': 9898,\n",
       " 'cut': 5609,\n",
       " 'you': 26577,\n",
       " 'know': 13172,\n",
       " 're': 18997,\n",
       " 'for': 9170,\n",
       " 'treat': 24421,\n",
       " 'br': 2717,\n",
       " 'pioneers': 17592,\n",
       " 'big': 2205,\n",
       " 'wave': 25894,\n",
       " 'surfing': 23160,\n",
       " 'are': 1123,\n",
       " 'portrayed': 17919,\n",
       " 'very': 25459,\n",
       " 'evocative': 8122,\n",
       " 'lost': 13999,\n",
       " 'era': 7930,\n",
       " 'nevermind': 15938,\n",
       " 'fact': 8438,\n",
       " 'no': 16055,\n",
       " 'knows': 13179,\n",
       " 'how': 11384,\n",
       " 'these': 23839,\n",
       " 'guys': 10463,\n",
       " 'made': 14216,\n",
       " 'living': 13849,\n",
       " 'much': 15540,\n",
       " 'less': 13614,\n",
       " 'took': 24148,\n",
       " 'care': 3390,\n",
       " 'like': 13727,\n",
       " 'medical': 14778,\n",
       " 'use': 25260,\n",
       " 'old': 16413,\n",
       " 'clips': 4233,\n",
       " 'throughout': 23937,\n",
       " 'was': 25847,\n",
       " 'masterfully': 14582,\n",
       " 'done': 6909,\n",
       " 'they': 23846,\n",
       " 'showing': 21345,\n",
       " 'on': 16453,\n",
       " 'some': 21997,\n",
       " 'network': 15926,\n",
       " 'well': 26001,\n",
       " 'crap': 5293,\n",
       " 'while': 26066,\n",
       " 'as': 1240,\n",
       " 'bad': 1635,\n",
       " 'movies': 15521,\n",
       " 'show': 21331,\n",
       " 'sci': 20671,\n",
       " 'fi': 8761,\n",
       " 'but': 3144,\n",
       " 'still': 22621,\n",
       " 'fairly': 8470,\n",
       " 'large': 13383,\n",
       " 'pile': 17547,\n",
       " 'acting': 200,\n",
       " 'passable': 17065,\n",
       " 'plot': 17733,\n",
       " 'writing': 26460,\n",
       " 'sub': 22911,\n",
       " 'standard': 22455,\n",
       " 'pacing': 16832,\n",
       " 'entirely': 7870,\n",
       " 'too': 24147,\n",
       " 'slow': 21751,\n",
       " 'every': 8102,\n",
       " 'minute': 15110,\n",
       " 'movie': 15518,\n",
       " 'feels': 8674,\n",
       " 'part': 17029,\n",
       " 'where': 26054,\n",
       " 'wrapping': 26422,\n",
       " 'before': 1987,\n",
       " 'credits': 5356,\n",
       " 'peak': 17192,\n",
       " 'denouement': 6116,\n",
       " 'also': 665,\n",
       " 'portions': 17910,\n",
       " 'cast': 3509,\n",
       " 'look': 13946,\n",
       " 'age': 435,\n",
       " 'range': 18893,\n",
       " 'playing': 17691,\n",
       " 'whole': 26115,\n",
       " 'thing': 23857,\n",
       " 'predictable': 18078,\n",
       " 'boring': 2621,\n",
       " 'worthy': 26404,\n",
       " 'being': 2028,\n",
       " 'watched': 25870,\n",
       " 'save': 20535,\n",
       " 'your': 26584,\n",
       " 'time': 24020,\n",
       " 'even': 8088,\n",
       " 'worth': 26401,\n",
       " 'takes': 23438,\n",
       " 'free': 9371,\n",
       " 'blazing': 2351,\n",
       " 'saddles': 20352,\n",
       " 'fight': 8794,\n",
       " 'two': 24676,\n",
       " 'estranged': 8031,\n",
       " 'brothers': 2936,\n",
       " 'dennis': 6112,\n",
       " 'quaid': 18680,\n",
       " 'howard': 11385,\n",
       " 'both': 2650,\n",
       " 'whom': 26120,\n",
       " 'can': 3282,\n",
       " 'ignite': 11598,\n",
       " 'fires': 8888,\n",
       " 'mentally': 14870,\n",
       " 'square': 22362,\n",
       " 'over': 16713,\n",
       " 'childhood': 3906,\n",
       " 'differences': 6418,\n",
       " 'dippy': 6494,\n",
       " 'love': 14023,\n",
       " 'interest': 12279,\n",
       " 'debra': 5857,\n",
       " 'winger': 26224,\n",
       " 'caught': 3570,\n",
       " 'middle': 14987,\n",
       " 'director': 6506,\n",
       " 'glenn': 9937,\n",
       " 'gordon': 10066,\n",
       " 'caron': 3443,\n",
       " 'tv': 24643,\n",
       " 'whiz': 26109,\n",
       " 'kid': 13051,\n",
       " 'behind': 2023,\n",
       " 'moonlighting': 15380,\n",
       " 'smothers': 21828,\n",
       " 'darkly': 5744,\n",
       " 'textured': 23774,\n",
       " 'vince': 25558,\n",
       " 'gilligan': 9864,\n",
       " 'screenplay': 20746,\n",
       " 'presentation': 18142,\n",
       " 'so': 21907,\n",
       " 'slick': 21709,\n",
       " 'resembles': 19598,\n",
       " 'an': 788,\n",
       " 'entry': 7883,\n",
       " 'from': 9446,\n",
       " 'enthusiastic': 7864,\n",
       " 'student': 22850,\n",
       " 'fifteen': 8790,\n",
       " 'million': 15044,\n",
       " 'dollar': 6874,\n",
       " 'grant': 10171,\n",
       " 'prickly': 18215,\n",
       " 'energy': 7759,\n",
       " 'commercial': 4525,\n",
       " 'feature': 8649,\n",
       " 'shapeless': 21139,\n",
       " 'which': 26063,\n",
       " 'brings': 2875,\n",
       " 'nothing': 16151,\n",
       " 'characters': 3749,\n",
       " 'except': 8170,\n",
       " 'kooky': 13192,\n",
       " 'eccentricities': 7387,\n",
       " 'aren': 1126,\n",
       " 'functions': 9503,\n",
       " 'barely': 1774,\n",
       " 'released': 19370,\n",
       " 'theaters': 23797,\n",
       " 'disaster': 6544,\n",
       " 'although': 683,\n",
       " 'strictly': 22791,\n",
       " 'example': 8153,\n",
       " 'substance': 22947,\n",
       " 'does': 6850,\n",
       " 'good': 10039,\n",
       " 'only': 16464,\n",
       " 'looks': 13953,\n",
       " 'truly': 24556,\n",
       " 'perplexed': 17354,\n",
       " 'encouraged': 7721,\n",
       " 'by': 3170,\n",
       " 'positive': 17934,\n",
       " 'here': 10981,\n",
       " 'looking': 13951,\n",
       " 'forward': 9276,\n",
       " 'watching': 25875,\n",
       " 'mistake': 15196,\n",
       " 've': 25379,\n",
       " 'films': 8830,\n",
       " 'worst': 26400,\n",
       " 'them': 23808,\n",
       " 'awful': 1562,\n",
       " 'almost': 649,\n",
       " 'editing': 7426,\n",
       " 'storyline': 22706,\n",
       " 'soundtrack': 22076,\n",
       " 'song': 22017,\n",
       " 'lame': 13320,\n",
       " 'country': 5198,\n",
       " 'tune': 24610,\n",
       " 'played': 17683,\n",
       " 'four': 9292,\n",
       " 'times': 24028,\n",
       " 'cheap': 3805,\n",
       " 'nasty': 15771,\n",
       " 'extreme': 8387,\n",
       " 'rarely': 18938,\n",
       " 'happy': 10654,\n",
       " 'see': 20860,\n",
       " 'end': 7728,\n",
       " 'prevents': 18205,\n",
       " 'me': 14734,\n",
       " 'giving': 9901,\n",
       " 'score': 20690,\n",
       " 'harvey': 10728,\n",
       " 'keitel': 12999,\n",
       " 'far': 8532,\n",
       " 'best': 2143,\n",
       " 'performance': 17307,\n",
       " 'at': 1357,\n",
       " 'least': 13518,\n",
       " 'seems': 20875,\n",
       " 'making': 14303,\n",
       " 'bit': 2276,\n",
       " 'effort': 7463,\n",
       " 'when': 26052,\n",
       " 'finally': 8839,\n",
       " 'had': 10498,\n",
       " 'opportunity': 16517,\n",
       " 'zombie': 26669,\n",
       " 'europe': 8063,\n",
       " 'import': 11761,\n",
       " 'region': 19286,\n",
       " 'japanese': 12622,\n",
       " 'dvd': 7298,\n",
       " 'blown': 2443,\n",
       " 'away': 1553,\n",
       " 'just': 12894,\n",
       " 'entertaining': 7854,\n",
       " 'epic': 7903,\n",
       " 'transfer': 24327,\n",
       " 'immaculate': 11678,\n",
       " 'going': 10011,\n",
       " 'unless': 25052,\n",
       " 'anchor': 815,\n",
       " 'bay': 1896,\n",
       " 'gets': 9807,\n",
       " 'hold': 11186,\n",
       " 'gore': 10068,\n",
       " 'stands': 22461,\n",
       " 'really': 19037,\n",
       " 'appreciate': 1054,\n",
       " 'excellent': 8167,\n",
       " 'makeup': 14301,\n",
       " 'fx': 9549,\n",
       " 'sound': 22071,\n",
       " 'terrific': 23733,\n",
       " 'channel': 3718,\n",
       " 'dolby': 6869,\n",
       " 'if': 11594,\n",
       " 'cheesy': 3848,\n",
       " 'music': 15623,\n",
       " 'actually': 219,\n",
       " 'effective': 7455,\n",
       " 'effects': 7458,\n",
       " 'sounded': 22072,\n",
       " 'adds': 258,\n",
       " 'overall': 16719,\n",
       " 'enjoyment': 7800,\n",
       " 'realized': 19034,\n",
       " 'blood': 2414,\n",
       " 'flows': 9078,\n",
       " 'extremely': 8388,\n",
       " 'brutal': 2960,\n",
       " 'exploding': 8313,\n",
       " 'head': 10807,\n",
       " 'shots': 21314,\n",
       " 'puss': 18655,\n",
       " 'filled': 8813,\n",
       " 'mega': 14802,\n",
       " 'cleaver': 4176,\n",
       " 'zombies': 26670,\n",
       " 'throat': 23928,\n",
       " 'woman': 26323,\n",
       " 'burned': 3101,\n",
       " 'extremities': 8392,\n",
       " 'come': 4473,\n",
       " 'did': 6402,\n",
       " 'nt': 16187,\n",
       " 'burn': 3100,\n",
       " 'munching': 15588,\n",
       " 'babies': 1593,\n",
       " 'track': 24267,\n",
       " 'hardcore': 10666,\n",
       " 'action': 201,\n",
       " 'fans': 8524,\n",
       " 'especially': 8000,\n",
       " 'italian': 12541,\n",
       " 'kind': 13089,\n",
       " 'set': 21030,\n",
       " 'pieces': 17531,\n",
       " 'cinematography': 4061,\n",
       " 'found': 9286,\n",
       " 'think': 23860,\n",
       " 'people': 17273,\n",
       " 'don': 6902,\n",
       " 'give': 9896,\n",
       " 'enough': 7819,\n",
       " 'credit': 5353,\n",
       " 'clean': 4160,\n",
       " 'print': 18245,\n",
       " 'horrendous': 11319,\n",
       " 'pirate': 17599,\n",
       " 'copy': 5084,\n",
       " 'experience': 8283,\n",
       " 'lets': 13626,\n",
       " 'second': 20829,\n",
       " 'realize': 19033,\n",
       " 'inconsistent': 11885,\n",
       " 'plotwise': 17740,\n",
       " 'dubbing': 7187,\n",
       " 'horrible': 11321,\n",
       " 'stiff': 22613,\n",
       " 'sense': 20944,\n",
       " 'irreverence': 12496,\n",
       " 'celebrated': 3609,\n",
       " 'grand': 10154,\n",
       " 'fashion': 8570,\n",
       " 'charm': 3773,\n",
       " 'horror': 11329,\n",
       " 'make': 14294,\n",
       " 'purpose': 18634,\n",
       " 'accidental': 114,\n",
       " 'highest': 11052,\n",
       " 'order': 16560,\n",
       " 'laughs': 13434,\n",
       " 'would': 26405,\n",
       " 'straight': 22714,\n",
       " 'arrow': 1205,\n",
       " 'result': 19698,\n",
       " 'cult': 5537,\n",
       " 'classic': 4131,\n",
       " 'thumbs': 23956,\n",
       " 'nose': 16132,\n",
       " 'any': 973,\n",
       " 'all': 605,\n",
       " 'traditional': 24283,\n",
       " 'standards': 22456,\n",
       " 'tons': 24145,\n",
       " 'sequences': 20986,\n",
       " 'exotic': 8261,\n",
       " 'locales': 13882,\n",
       " 'design': 6222,\n",
       " 'sometimes': 22011,\n",
       " 'great': 10217,\n",
       " 'wonderfully': 26334,\n",
       " 'interesting': 12281,\n",
       " 'beautiful': 1941,\n",
       " 'women': 26328,\n",
       " 'who': 26110,\n",
       " 'kick': 13045,\n",
       " 'butt': 3151,\n",
       " 'creepy': 5371,\n",
       " 'always': 693,\n",
       " 'go': 9986,\n",
       " 'wrong': 26463,\n",
       " 'test': 23754,\n",
       " 'stan': 22452,\n",
       " 'ollie': 16428,\n",
       " 'spinning': 22234,\n",
       " 'graves': 10207,\n",
       " 'won': 26330,\n",
       " 'bother': 2651,\n",
       " 'listing': 13816,\n",
       " 'fundamental': 9505,\n",
       " 'flaws': 8983,\n",
       " 'obvious': 16312,\n",
       " 'without': 26288,\n",
       " 'saying': 20554,\n",
       " 'small': 21780,\n",
       " 'new': 15941,\n",
       " 'adventures': 345,\n",
       " 'laurel': 13445,\n",
       " 'hardy': 10678,\n",
       " 'despite': 6247,\n",
       " 'stars': 22490,\n",
       " 'dead': 5816,\n",
       " 'thirty': 23876,\n",
       " 'years': 26539,\n",
       " 'little': 13833,\n",
       " 'idea': 11557,\n",
       " 'actors': 213,\n",
       " 'buffoons': 3008,\n",
       " 'happen': 10644,\n",
       " 'called': 3236,\n",
       " 'might': 15001,\n",
       " 'worked': 26368,\n",
       " 'try': 24576,\n",
       " 'casting': 3513,\n",
       " 'impersonate': 11730,\n",
       " 'duo': 7263,\n",
       " 'okay': 16410,\n",
       " 'claim': 4101,\n",
       " 'nephews': 15907,\n",
       " 'same': 20422,\n",
       " 'bronson': 2916,\n",
       " 'pinchot': 17568,\n",
       " 'forget': 9218,\n",
       " 'wacky': 25709,\n",
       " 'foreigner': 9196,\n",
       " 'cousin': 5226,\n",
       " 'larry': 13391,\n",
       " 'schtick': 20664,\n",
       " 'perfect': 17300,\n",
       " 'strangers': 22732,\n",
       " 'him': 11089,\n",
       " 'true': 24551,\n",
       " 'romance': 20091,\n",
       " 'though': 23893,\n",
       " 'stinks': 22638,\n",
       " 'probably': 18275,\n",
       " 'fault': 8603,\n",
       " 'support': 23129,\n",
       " 'better': 2164,\n",
       " 'material': 14607,\n",
       " 'desperate': 6240,\n",
       " 'money': 15321,\n",
       " 'those': 23891,\n",
       " 'find': 8850,\n",
       " 'difficult': 6425,\n",
       " 'master': 14579,\n",
       " 'english': 7779,\n",
       " 'accent': 98,\n",
       " 'cause': 3572,\n",
       " 'helped': 10940,\n",
       " 'what': 26041,\n",
       " 'welsh': 26007,\n",
       " 'iranian': 12462,\n",
       " 'pakistani': 16885,\n",
       " 'trademark': 24276,\n",
       " 'close': 4249,\n",
       " 'overdone': 16732,\n",
       " 'point': 17789,\n",
       " 'tedium': 23614,\n",
       " 'write': 26454,\n",
       " 'home': 11218,\n",
       " 'gailard': 9576,\n",
       " 'sartain': 20484,\n",
       " 'slightly': 21717,\n",
       " 'worse': 26392,\n",
       " 'stepping': 22580,\n",
       " 'dog': 6853,\n",
       " 'dirt': 6512,\n",
       " 'or': 16543,\n",
       " 'remember': 19429,\n",
       " 'originals': 16598,\n",
       " 'split': 22270,\n",
       " 'timing': 24031,\n",
       " 'intuitive': 12395,\n",
       " 'teamwork': 23580,\n",
       " 'innate': 12107,\n",
       " 'absent': 63,\n",
       " 'altogether': 686,\n",
       " 'replaced': 19512,\n",
       " 'stupid': 22884,\n",
       " 'men': 14856,\n",
       " 'jokes': 12770,\n",
       " 'park': 17010,\n",
       " 'bus': 3120,\n",
       " 'gaps': 9632,\n",
       " 'whereas': 26056,\n",
       " 'plots': 17737,\n",
       " 'summed': 23066,\n",
       " 'couple': 5205,\n",
       " 'panels': 16928,\n",
       " 'long': 13936,\n",
       " 'winded': 26209,\n",
       " 'mummy': 15584,\n",
       " 'hokum': 11183,\n",
       " 'lousy': 14019,\n",
       " 'title': 24070,\n",
       " 'mixed': 15220,\n",
       " 'boys': 2713,\n",
       " 'fraternity': 9350,\n",
       " 'scenario': 20610,\n",
       " 'single': 21519,\n",
       " 'safe': 20365,\n",
       " 'bet': 2147,\n",
       " 'nadir': 15698,\n",
       " 'leagues': 13496,\n",
       " 'ahead': 473,\n",
       " 'maybe': 14664,\n",
       " 'major': 14290,\n",
       " 'problem': 18280,\n",
       " 'were': 26017,\n",
       " 'sort': 22059,\n",
       " 'themselves': 23814,\n",
       " 'own': 16809,\n",
       " 'accents': 100,\n",
       " 'felt': 8693,\n",
       " 'unforced': 24961,\n",
       " 'opposed': 16519,\n",
       " 'contrived': 5000,\n",
       " 'caricatures': 3412,\n",
       " 'since': 21506,\n",
       " 'do': 6819,\n",
       " 'many': 14430,\n",
       " 'gonna': 10034,\n",
       " 'standing': 22457,\n",
       " 'my': 15658,\n",
       " 'friend': 9415,\n",
       " 'stop': 22686,\n",
       " 'notable': 16141,\n",
       " 'moment': 15300,\n",
       " 'reference': 19227,\n",
       " 'bozo': 2715,\n",
       " 'clown': 4275,\n",
       " 'cartoon': 3471,\n",
       " 'shared': 21147,\n",
       " 'harmon': 10695,\n",
       " 'comic': 4495,\n",
       " 'course': 5213,\n",
       " 'bought': 2663,\n",
       " 'name': 15719,\n",
       " 'copyright': 5087,\n",
       " 'disconcerting': 6567,\n",
       " 'after': 419,\n",
       " 'co': 4301,\n",
       " 'producer': 18309,\n",
       " 'travesty': 24405,\n",
       " 'questions': 18721,\n",
       " 'abound': 50,\n",
       " 'fart': 8556,\n",
       " 'gags': 9574,\n",
       " 'alive': 604,\n",
       " 'today': 24090,\n",
       " 'glass': 9924,\n",
       " 'mummies': 15582,\n",
       " 'broken': 2913,\n",
       " 'bottles': 2658,\n",
       " 'smacked': 21777,\n",
       " 'genitals': 9757,\n",
       " 'spear': 22157,\n",
       " 'cgi': 3668,\n",
       " 'finale': 8838,\n",
       " 'let': 13622,\n",
       " 'hope': 11287,\n",
       " 'once': 16456,\n",
       " 'disbelief': 6548,\n",
       " 'terrible': 23730,\n",
       " 'why': 26134,\n",
       " 'first': 8894,\n",
       " 'place': 17632,\n",
       " 'makers': 14298,\n",
       " 'possibly': 17950,\n",
       " 'absolute': 65,\n",
       " 'grew': 10256,\n",
       " 'sick': 21399,\n",
       " 'five': 8919,\n",
       " 'minutes': 15111,\n",
       " 'fun': 9499,\n",
       " 'having': 10777,\n",
       " 'trapped': 24375,\n",
       " 'vice': 25481,\n",
       " 'red': 19173,\n",
       " 'hot': 11353,\n",
       " 'poker': 17811,\n",
       " 'stinging': 22632,\n",
       " 'forcibly': 9183,\n",
       " 'inserted': 12143,\n",
       " 'back': 1609,\n",
       " 'passage': 17066,\n",
       " 'judy': 12848,\n",
       " 'holliday': 11199,\n",
       " 'struck': 22829,\n",
       " 'gold': 10015,\n",
       " 'george': 9781,\n",
       " 'cukor': 5528,\n",
       " 'version': 25453,\n",
       " 'born': 2625,\n",
       " 'yesterday': 26552,\n",
       " 'her': 10970,\n",
       " 'career': 3392,\n",
       " 'consisted': 4873,\n",
       " 'trying': 24577,\n",
       " 'allow': 629,\n",
       " 'strike': 22796,\n",
       " 'again': 431,\n",
       " 'happened': 10645,\n",
       " 'yet': 26554,\n",
       " 'variation': 25354,\n",
       " 'dumb': 7229,\n",
       " 'blonde': 2410,\n",
       " 'everything': 8108,\n",
       " 'warmed': 25818,\n",
       " 'half': 10541,\n",
       " 'hearted': 10854,\n",
       " 'jack': 12564,\n",
       " 'lemmon': 13575,\n",
       " 'believe': 2048,\n",
       " 'role': 20077,\n",
       " 'muster': 15639,\n",
       " 'enliven': 7812,\n",
       " 'recycled': 19170,\n",
       " 'audience': 1445,\n",
       " 'will': 26177,\n",
       " 'virtually': 25593,\n",
       " 'beginning': 2003,\n",
       " 'mostly': 15457,\n",
       " 'sits': 21558,\n",
       " 'around': 1179,\n",
       " 'waiting': 25732,\n",
       " 'catch': 3535,\n",
       " 'enamored': 7706,\n",
       " 'll': 13856,\n",
       " 'otherwise': 16629,\n",
       " 'wouldn': 26407,\n",
       " 'grade': 10130,\n",
       " 'weakest': 25920,\n",
       " 'soft': 21950,\n",
       " 'porn': 17899,\n",
       " 'somebody': 21998,\n",
       " 'wrote': 26468,\n",
       " 'story': 22703,\n",
       " 'changes': 3716,\n",
       " 'mike': 15008,\n",
       " 'wimp': 26200,\n",
       " 'moron': 15423,\n",
       " 'didn': 6404,\n",
       " 'want': 25793,\n",
       " 'take': 23434,\n",
       " 'shower': 21339,\n",
       " 'bride': 2840,\n",
       " 'toni': 24140,\n",
       " 'threesome': 23915,\n",
       " 'french': 9385,\n",
       " 'photographer': 17481,\n",
       " 'jan': 12607,\n",
       " 'short': 21298,\n",
       " 'hate': 10748,\n",
       " 'core': 5092,\n",
       " 'man': 14349,\n",
       " 'girl': 9882,\n",
       " 'hour': 11368,\n",
       " 'alot': 657,\n",
       " 'longer': 13938,\n",
       " 'scenes': 20617,\n",
       " 'face': 8425,\n",
       " 'final': 8837,\n",
       " 'season': 20816,\n",
       " 'seasons': 20819,\n",
       " 'enjoyed': 7798,\n",
       " 'dry': 7176,\n",
       " 'spells': 22199,\n",
       " 'last': 13404,\n",
       " 'judge': 12834,\n",
       " 'across': 197,\n",
       " 'inferior': 12007,\n",
       " 'unfair': 24943,\n",
       " 'assessment': 1304,\n",
       " 'because': 1949,\n",
       " 'its': 12548,\n",
       " 'day': 5798,\n",
       " 'brilliant': 2868,\n",
       " 'sitcom': 21553,\n",
       " 'bygone': 3172,\n",
       " 'lived': 13838,\n",
       " 'weren': 26018,\n",
       " 'different': 6419,\n",
       " 'we': 25914,\n",
       " 'modern': 15261,\n",
       " 'right': 19923,\n",
       " 'ignoring': 11607,\n",
       " 'topher': 24162,\n",
       " 'grace': 10122,\n",
       " 'eric': 7943,\n",
       " 'forman': 9238,\n",
       " 'horny': 11318,\n",
       " 'geek': 9708,\n",
       " 'teenager': 23619,\n",
       " 'perpetual': 17348,\n",
       " 'donna': 6914,\n",
       " 'laura': 13444,\n",
       " 'prepon': 18128,\n",
       " 'feminist': 8699,\n",
       " 'next': 15965,\n",
       " 'door': 6937,\n",
       " 'friends': 9417,\n",
       " 'danny': 5722,\n",
       " 'masterson': 14588,\n",
       " 'hyde': 11512,\n",
       " 'mila': 15016,\n",
       " 'kunis': 13240,\n",
       " 'jackie': 12570,\n",
       " 'fez': 8759,\n",
       " 'ashton': 1253,\n",
       " 'kutcher': 13249,\n",
       " 'kelso': 13007,\n",
       " 'fantastic': 8528,\n",
       " 'performances': 17308,\n",
       " 'episode': 7909,\n",
       " 'foreign': 9195,\n",
       " 'exchange': 8183,\n",
       " 'promiscuous': 18376,\n",
       " 'get': 9805,\n",
       " 'figure': 8802,\n",
       " 'another': 920,\n",
       " 'dimension': 6460,\n",
       " 'entertainment': 7856,\n",
       " 'jo': 12731,\n",
       " 'rupp': 20284,\n",
       " 'kurtwood': 13245,\n",
       " 'smith': 21811,\n",
       " 'phenomenal': 17441,\n",
       " 'parents': 17005,\n",
       " 'kitty': 13130,\n",
       " 'formidable': 9246,\n",
       " 'sweet': 23276,\n",
       " 'mrs': 15534,\n",
       " 'brady': 2729,\n",
       " 'meets': 14800,\n",
       " 'marie': 14469,\n",
       " 'exists': 8252,\n",
       " 'mainly': 14281,\n",
       " 'scare': 20584,\n",
       " 'stark': 22482,\n",
       " 'tanya': 23497,\n",
       " 'roberts': 20025,\n",
       " 'play': 17680,\n",
       " 'opposite': 16522,\n",
       " 'each': 7330,\n",
       " 'chauvinistic': 3798,\n",
       " 'likable': 13726,\n",
       " 'bob': 2480,\n",
       " 'tommy': 24129,\n",
       " 'chong': 3960,\n",
       " 'occasional': 16317,\n",
       " 'appearances': 1026,\n",
       " 'leo': 13596,\n",
       " 'stoner': 22673,\n",
       " 'acts': 216,\n",
       " 'father': 8592,\n",
       " 'apart': 986,\n",
       " 'anachronistic': 792,\n",
       " 'errors': 7967,\n",
       " 'pop': 17877,\n",
       " 'frequently': 9393,\n",
       " 'top': 24161,\n",
       " 'lessons': 13619,\n",
       " 'deplorable': 6154,\n",
       " 'amazing': 711,\n",
       " 'spot': 22308,\n",
       " 'direction': 6502,\n",
       " 'feel': 8670,\n",
       " 'vibe': 25473,\n",
       " 'alright': 664,\n",
       " 'liked': 13729,\n",
       " 'budding': 2994,\n",
       " 'ingrid': 12065,\n",
       " 'bergman': 2117,\n",
       " 'mel': 14808,\n",
       " 'ferrer': 8722,\n",
       " 'cute': 5611,\n",
       " 'wanted': 25794,\n",
       " 'inevitable': 11980,\n",
       " 'however': 11387,\n",
       " 'switched': 23311,\n",
       " 'fiancé': 8764,\n",
       " 'fell': 8682,\n",
       " 'instead': 12191,\n",
       " 'excruciatingly': 8203,\n",
       " 'dopey': 6947,\n",
       " 'parallel': 16979,\n",
       " 'emerge': 7642,\n",
       " 'ruin': 20250,\n",
       " 'boorish': 2585,\n",
       " 'son': 22015,\n",
       " 'military': 15030,\n",
       " 'vying': 25702,\n",
       " 'maid': 14271,\n",
       " 'attention': 1412,\n",
       " 'looked': 13948,\n",
       " 'subplot': 22939,\n",
       " 'boat': 2477,\n",
       " 'elegance': 7530,\n",
       " 'portion': 17909,\n",
       " 'beyond': 2183,\n",
       " 'attempt': 1400,\n",
       " 'renoir': 19474,\n",
       " 'recapture': 19080,\n",
       " 'success': 22984,\n",
       " 'rules': 20260,\n",
       " 'game': 9606,\n",
       " 'similar': 21478,\n",
       " 'switches': 23312,\n",
       " 'estate': 8020,\n",
       " 'huge': 11407,\n",
       " 'fan': 8507,\n",
       " 'elena': 7537,\n",
       " 'appreciating': 1057,\n",
       " 'artistry': 1231,\n",
       " 'nuances': 16193,\n",
       " 'original': 16595,\n",
       " 'pinter': 17585,\n",
       " 'knee': 13148,\n",
       " 'shall': 21114,\n",
       " 'bow': 2685,\n",
       " 'nobel': 16058,\n",
       " 'literature': 13828,\n",
       " 'prize': 18269,\n",
       " 'acceptance': 106,\n",
       " 'speech': 22185,\n",
       " 'canned': 3310,\n",
       " 'numbers': 16209,\n",
       " 'sixth': 21570,\n",
       " 'form': 9236,\n",
       " 'anti': 937,\n",
       " 'tour': 24234,\n",
       " 'de': 5815,\n",
       " 'force': 9178,\n",
       " 'superb': 23094,\n",
       " 'decades': 5869,\n",
       " 'reputation': 19566,\n",
       " 'represents': 19544,\n",
       " 'piece': 17529,\n",
       " 'sleuth': 21703,\n",
       " 'masterpiece': 14585,\n",
       " 'yes': 26551,\n",
       " 'theatrical': 23800,\n",
       " 'confection': 4753,\n",
       " 'easy': 7373,\n",
       " 'central': 3644,\n",
       " 'device': 6324,\n",
       " 'work': 26365,\n",
       " 'stage': 22407,\n",
       " 'screen': 20741,\n",
       " 'terrifically': 23734,\n",
       " 'michael': 14968,\n",
       " 'caine': 3216,\n",
       " 'vanity': 25348,\n",
       " 'olivier': 16427,\n",
       " 'fill': 8812,\n",
       " 'shoes': 21276,\n",
       " 'couldn': 5176,\n",
       " 'bathroom': 1868,\n",
       " 'slippers': 21726,\n",
       " 'appropriately': 1070,\n",
       " 'named': 15720,\n",
       " 'distinctly': 6740,\n",
       " 'average': 1521,\n",
       " 'actor': 212,\n",
       " 'whose': 26132,\n",
       " 'real': 19020,\n",
       " 'recommendation': 19129,\n",
       " 'british': 2887,\n",
       " 'longevity': 13940,\n",
       " 'business': 3131,\n",
       " 'harry': 10715,\n",
       " 'palmer': 16901,\n",
       " 'carter': 3469,\n",
       " 'yer': 26550,\n",
       " 'lot': 14000,\n",
       " 'mate': 14605,\n",
       " 'wide': 26141,\n",
       " 'berth': 2134,\n",
       " 'stick': 22606,\n",
       " 'writer': 26455,\n",
       " 'john': 12752,\n",
       " 'hughes': 11411,\n",
       " 'covered': 5232,\n",
       " 'bases': 1831,\n",
       " 'usual': 25273,\n",
       " 'bitter': 2287,\n",
       " 'sunday': 23081,\n",
       " 'afternoon': 423,\n",
       " 'family': 8503,\n",
       " 'curly': 5573,\n",
       " 'sue': 23011,\n",
       " 'precocious': 18063,\n",
       " 'orphan': 16607,\n",
       " 'cared': 3391,\n",
       " 'infancy': 11994,\n",
       " 'bill': 2228,\n",
       " 'pair': 16877,\n",
       " 'live': 13837,\n",
       " 'wits': 26297,\n",
       " 'travel': 24393,\n",
       " 'us': 25256,\n",
       " 'fate': 8588,\n",
       " 'matches': 14603,\n",
       " 'pretty': 18196,\n",
       " 'yuppie': 26609,\n",
       " 'lawyer': 13465,\n",
       " 'rest': 19675,\n",
       " 'kids': 13066,\n",
       " 'relate': 19351,\n",
       " 'heroine': 10999,\n",
       " 'year': 26533,\n",
       " 'went': 26014,\n",
       " 'pepsi': 17278,\n",
       " 'commercials': 4528,\n",
       " 'supposed': 23137,\n",
       " 'she': 21174,\n",
       " 'urged': 25245,\n",
       " 'school': 20653,\n",
       " 'vocabulary': 25643,\n",
       " 'suggests': 23034,\n",
       " 'older': 16414,\n",
       " 'alone': 651,\n",
       " 'plenty': 17723,\n",
       " 'slap': 21651,\n",
       " 'fists': 8910,\n",
       " 'punching': 18590,\n",
       " 'fat': 8581,\n",
       " 'formula': 9249,\n",
       " 'aimed': 494,\n",
       " 'young': 26578,\n",
       " 'heartwarming': 10864,\n",
       " 'surprises': 23176,\n",
       " 'prepared': 18125,\n",
       " 'shed': 21178,\n",
       " 'tear': 23581,\n",
       " 'anime': 878,\n",
       " 'series': 21002,\n",
       " 'starts': 22502,\n",
       " 'exciting': 8192,\n",
       " ...}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = []\n",
    "for key in words_train:\n",
    "    y_train.append(int(key[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = np.array(y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# train_val_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "训练数据大小为： (24500, 26692)\n",
      "标签大小为: (24500,)\n"
     ]
    }
   ],
   "source": [
    "#  train_x, y_label; x_val, y_val\n",
    "labels = y_train\n",
    "VALIDATION_SPLIT = 0.2\n",
    "print(\"训练数据大小为：\", X_train.shape)  # (24500, 500)\n",
    "print(\"标签大小为:\", labels.shape)  # (24500, 1)\n",
    " \n",
    "# 将训练数据划分为训练集和验证集\n",
    "indices = np.arange(X_train.shape[0])\n",
    "np.random.seed(10)\n",
    "np.random.shuffle(indices)  # 打乱数据\n",
    "data_shuffle = X_train[indices]\n",
    "labels_shuffle = labels[indices]\n",
    " \n",
    "num_validation_samples = int(VALIDATION_SPLIT * X_train.shape[0])\n",
    "#num_test_samples = int(TEST_SPLIT * data.shape[0])\n",
    " \n",
    "# 训练数据\n",
    "train_x= data_shuffle[:-num_validation_samples]\n",
    "y_label = labels_shuffle[:-num_validation_samples]\n",
    " \n",
    "# 验证数据\n",
    "x_val = data_shuffle[-num_validation_samples:]\n",
    "y_val = labels_shuffle[-num_validation_samples:]\n",
    "#test data\n",
    "# x_test = data_shuffle[-num_test_samples:]\n",
    "# y_test = labels_shuffle[-num_test_samples:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5042857142857143\n"
     ]
    }
   ],
   "source": [
    "pos = 0\n",
    "for i in y_val:\n",
    "    if i==1:\n",
    "        pos+=1\n",
    "print(pos/y_val.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SVM (linear)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---kernel: linear --c: 0.1\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 0.30000000000000004\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 0.5000000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 0.7000000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 0.9000000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 1.1000000000000003\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 1.3000000000000003\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 1.5000000000000004\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 1.7000000000000004\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 1.9000000000000004\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 2.1000000000000005\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 2.3000000000000007\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 2.5000000000000004\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 2.7000000000000006\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 2.900000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 3.1000000000000005\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 3.3000000000000007\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 3.500000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 3.7000000000000006\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 3.900000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 4.1000000000000005\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 4.300000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 4.500000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 4.7\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 4.9\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 5.1000000000000005\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 5.300000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 5.500000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 5.700000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 5.9\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 6.1000000000000005\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 6.300000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 6.500000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 6.700000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 6.900000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 7.100000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 7.300000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 7.500000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 7.700000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 7.900000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 8.100000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 8.3\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 8.500000000000002\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 8.700000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 8.900000000000002\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 9.100000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 9.3\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 9.500000000000002\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 9.700000000000001\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "---kernel: linear --c: 9.900000000000002\n",
      "[LibSVM]--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    }
   ],
   "source": [
    "# simple SVC\n",
    "from sklearn.svm import SVC\n",
    "clf_list = []\n",
    "clf_val = []\n",
    "clf_train = []\n",
    "clf_auc = []\n",
    "for k in ['linear']:\n",
    "    for i in np.arange(0.1,10.1,0.2):\n",
    "        print('---kernel:',k,'--c:',i)\n",
    "        clf = SVC(C = i, random_state=0, kernel = k, verbose=1)\n",
    "        clf.fit(train_x,y_label) \n",
    "        print('--fit--finish!!!')\n",
    "        y_pred = clf.predict(x_val)\n",
    "        print('--predict finish----')\n",
    "        val_acc =  clf.score(x_val, y_val)\n",
    "        print('----scores----')\n",
    "        auc = roc_auc_score(y_val, y_pred)\n",
    "        clf_list.append(clf)\n",
    "        clf_val.append(val_acc)\n",
    "        clf_train.append(train_acc)\n",
    "        clf_auc.append(auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_auc.index(0.8896653631695389)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=0.5000000000000001, cache_size=200, class_weight=None, coef0=0.0,\n",
       "    decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
       "    kernel='linear', max_iter=-1, probability=False, random_state=0,\n",
       "    shrinking=True, tol=0.001, verbose=1)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_list[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibSVM]0.889795918367347\n",
      "0.9457142857142857\n",
      "[[2124  305]\n",
      " [ 235 2236]]\n"
     ]
    }
   ],
   "source": [
    "# linear SVM\n",
    "from sklearn.svm import SVC\n",
    "clf = SVC(C=0.5000000000000001, cache_size=200, class_weight=None, coef0=0.0,\n",
    "    decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
    "    kernel='linear', max_iter=-1, probability=True, random_state=0,\n",
    "    shrinking=True, tol=0.001, verbose=1).fit(train_x,y_label)\n",
    "y_pred = clf.predict(x_val)\n",
    "val_acc =  clf.score(x_val, y_val)\n",
    "print(val_acc)\n",
    "train_acc =  clf.score(train_x,y_label)\n",
    "print(train_acc) \n",
    "confusion_array_clf = confusion_matrix(y_val, y_pred)\n",
    "print(confusion_array_clf)\n",
    "#        pred_0 pred_1\n",
    "#act_0   2124     305\n",
    "#act_1   235     2236"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\externals\\joblib\\__init__.py:15: DeprecationWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
      "  warnings.warn(msg, category=DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['submit\\\\train_model_SVM_linear_c0.5.m']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_prob = clf.predict_proba(X_test) \n",
    "\n",
    "np.savetxt(r'submit\\submission_SVM_linear_c0.5.txt',y_pred_prob[:,1], fmt='%.9f')\n",
    "from sklearn.externals import joblib\n",
    "joblib.dump(clf, r'submit\\train_model_SVM_linear_c0.5.m')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "clf = LogisticRegression(random_state=0, solver='sag',\n",
    "                               multi_class='ovr',C=0.8,verbose=1).fit(X_train[:20000,:], y_train[:20000])\n",
    "y_pred = clf.predict(X_train[20000:,:])\n",
    "y_pred_prob = clf.predict_proba(X_train[20000:,:]) \n",
    "acc = clf.score(X_train[20000:,:], y_train[20000:])\n",
    "print('test_acc:',acc)\n",
    "acc_train = clf.score(X_train[:20000,:], y_train[:20000])\n",
    "print('train_acc:',acc_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression\n",
    "* select the best C value\\solver\\penalty "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    }
   ],
   "source": [
    "# select the best C value\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "clf_list = []\n",
    "clf_val = []\n",
    "clf_train = []\n",
    "clf_auc = []\n",
    "c = []\n",
    "for i in np.arange(0.5,8,0.1):\n",
    "    for s in ['lbfgs','newton-cg','sag']:\n",
    "        clf = LogisticRegression(random_state=0, solver=s,multi_class='ovr',C=i)\n",
    "        c.append(i)\n",
    "        clf.fit(train_x,y_label) \n",
    "        print('--fit--finish!!!')\n",
    "        y_pred = clf.predict(x_val)\n",
    "        print('--predict finish----')\n",
    "        val_acc =  clf.score(x_val, y_val)\n",
    "        train_acc =  clf.score(train_x,y_label) \n",
    "        print('----scores----')\n",
    "        auc = roc_auc_score(y_val, y_pred)\n",
    "        clf_list.append(clf)\n",
    "        clf_val.append(val_acc)\n",
    "        clf_train.append(train_acc)\n",
    "        clf_auc.append(auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "123"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_auc.index(0.8892412134413057)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=4.599999999999999, class_weight=None, dual=False,\n",
       "                   fit_intercept=True, intercept_scaling=1, l1_ratio=None,\n",
       "                   max_iter=100, multi_class='ovr', n_jobs=None, penalty='l2',\n",
       "                   random_state=0, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_list[123]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.599999999999999"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c[123]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8887755102040816\n",
      "0.9719387755102041\n",
      "[[2121  308]\n",
      " [ 237 2234]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "clf = LogisticRegression(C=4.599999999999999, class_weight=None, dual=False,\n",
    "                   fit_intercept=True, intercept_scaling=1, l1_ratio=None,\n",
    "                   max_iter=100, multi_class='ovr', n_jobs=None, penalty='l2',\n",
    "                   random_state=0, solver='lbfgs', tol=0.0001, verbose=0,\n",
    "                   warm_start=False).fit(train_x,y_label)\n",
    "y_pred = clf.predict(x_val)\n",
    "val_acc =  clf.score(x_val, y_val)\n",
    "print(val_acc)\n",
    "train_acc =  clf.score(train_x,y_label)\n",
    "print(train_acc) \n",
    "confusion_array_clf = confusion_matrix(y_val, y_pred)\n",
    "print(confusion_array_clf)\n",
    "#        pred_0 pred_1\n",
    "#act_0   2121     308\n",
    "#act_1   237     2234"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22000, 2)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_prob = clf.predict_proba(X_test) \n",
    "\n",
    "np.savetxt('submission_logistic_c4.59999_lbfgs.txt',y_pred_prob[:,1], fmt='%.9f')\n",
    "from sklearn.externals import joblib\n",
    "joblib.dump(clf, r'train_model_logistic_c4.59999_lbfgs.m')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n",
      "--fit--finish!!!\n",
      "--predict finish----\n",
      "----scores----\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# select the best C value\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "clf_list = []\n",
    "clf_val = []\n",
    "clf_train = []\n",
    "clf_auc = []\n",
    "c = []\n",
    "for i in np.arange(0.5,8,0.1):\n",
    "    for s in ['l2','l1']:\n",
    "        clf = LogisticRegression(random_state=0, solver='liblinear',penalty=s,multi_class='ovr',C=i)\n",
    "        c.append(i)\n",
    "        clf.fit(train_x,y_label) \n",
    "        print('--fit--finish!!!')\n",
    "        y_pred = clf.predict(x_val)\n",
    "        print('--predict finish----')\n",
    "        val_acc =  clf.score(x_val, y_val)\n",
    "        train_acc =  clf.score(train_x,y_label) \n",
    "        print('----scores----')\n",
    "        auc = roc_auc_score(y_val, y_pred)\n",
    "        clf_list.append(clf)\n",
    "        clf_val.append(val_acc)\n",
    "        clf_train.append(train_acc)\n",
    "        clf_auc.append(auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "82"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_auc.index(0.8892412134413057)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8893877551020408\n",
      "LogisticRegression(C=4.599999999999999, class_weight=None, dual=False,\n",
      "                   fit_intercept=True, intercept_scaling=1, l1_ratio=None,\n",
      "                   max_iter=100, multi_class='ovr', n_jobs=None, penalty='l2',\n",
      "                   random_state=0, solver='liblinear', tol=0.0001, verbose=0,\n",
      "                   warm_start=False)\n"
     ]
    }
   ],
   "source": [
    "print(clf_val[82])\n",
    "print(clf_list[82])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8889795918367347\n",
      "0.9718367346938775\n",
      "[[2121  308]\n",
      " [ 236 2235]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "clf = LogisticRegression(C=4.599999999999999, class_weight=None, dual=False,\n",
    "                   fit_intercept=True, intercept_scaling=1, l1_ratio=None,\n",
    "                   max_iter=100, multi_class='ovr', n_jobs=None, penalty='l2',\n",
    "                   random_state=0, solver='liblinear', tol=0.0001, verbose=0,\n",
    "                   warm_start=False).fit(train_x, y_label)\n",
    "y_pred = clf.predict(x_val)\n",
    "val_acc =  clf.score(x_val, y_val)\n",
    "print(val_acc)\n",
    "train_acc =  clf.score(train_x,y_label)\n",
    "print(train_acc) \n",
    "confusion_array_clf = confusion_matrix(y_val, y_pred)\n",
    "print(confusion_array_clf)\n",
    "#        pred_0 pred_1\n",
    "#act_0   2121     308\n",
    "#act_1   236     2235\n",
    "y_pred_prob = clf.predict_proba(X_test) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['train_model_logistic_c4.599999_liblinear_l2.m']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.savetxt('submission_logistic_c4.599999_liblinear_l2.txt',y_pred_prob[:,1], fmt='%.9f')\n",
    "joblib.dump(clf, r'train_model_logistic_c4.599999_liblinear_l2.m')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "convergence after 24 epochs took 1 seconds\n",
      "train_acc: 0.9679183673469388\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    1.4s finished\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "clf = LogisticRegression(random_state=0, solver='sag',\n",
    "                               multi_class='ovr',C=4.4,verbose=1).fit(X_train, y_train)\n",
    "y_pred = clf.predict(X_test)\n",
    "y_pred_prob = clf.predict_proba(X_test) \n",
    "acc_train = clf.score(X_train, y_train)\n",
    "print('train_acc:',acc_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.score(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['train_model_logistic_c4.4_sag..m']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.savetxt('submission_logistic_c4.4_sag.txt',y_pred_prob[:,1], fmt='%.9f')\n",
    "joblib.dump(clf, r'train_model_logistic_c4.4_sag..m')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5042857142857143\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MLP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MLP128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "save_dir = r'D:\\v-yanx\\masijia\\text_mood_classification\\text_mood_classification\\weights_check'\n",
    "weight_path = 'model_{epoch:02d}-{val_acc:.4f}.hdf5'\n",
    "checkpoints = ModelCheckpoint(os.path.join(save_dir,weight_path), monitor='val_acc', verbose=0, save_best_only=False, save_weights_only=False, mode='max')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_12 (Dense)             (None, 128)               3416704   \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 3,416,833\n",
      "Trainable params: 3,416,833\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Multilayer Perceptron(MLP)\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from tensorflow.keras.models import Sequential\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(128, input_shape=(26692,), activation='relu'))\n",
    "model.add(Dropout(0.4))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "Train on 19600 samples, validate on 4900 samples\n",
      "Epoch 1/10\n",
      "19600/19600 [==============================] - 8s 399us/sample - loss: 0.4673 - acc: 0.8480 - val_loss: 0.3257 - val_acc: 0.8816\n",
      "Epoch 2/10\n",
      "19600/19600 [==============================] - 7s 363us/sample - loss: 0.2412 - acc: 0.9142 - val_loss: 0.2541 - val_acc: 0.8969\n",
      "Epoch 3/10\n",
      "19600/19600 [==============================] - 7s 365us/sample - loss: 0.1743 - acc: 0.9370 - val_loss: 0.2426 - val_acc: 0.8976\n",
      "Epoch 4/10\n",
      "19600/19600 [==============================] - 7s 367us/sample - loss: 0.1400 - acc: 0.9502 - val_loss: 0.2475 - val_acc: 0.8957\n",
      "Epoch 5/10\n",
      "19600/19600 [==============================] - 7s 366us/sample - loss: 0.1166 - acc: 0.9589 - val_loss: 0.2569 - val_acc: 0.8947\n",
      "Epoch 6/10\n",
      "19600/19600 [==============================] - 7s 367us/sample - loss: 0.0978 - acc: 0.9673 - val_loss: 0.2718 - val_acc: 0.8902\n",
      "Epoch 7/10\n",
      "19600/19600 [==============================] - 7s 367us/sample - loss: 0.0842 - acc: 0.9717 - val_loss: 0.2870 - val_acc: 0.8902\n",
      "Epoch 8/10\n",
      "19600/19600 [==============================] - 7s 373us/sample - loss: 0.0727 - acc: 0.9756 - val_loss: 0.3027 - val_acc: 0.8894\n",
      "Epoch 9/10\n",
      "19600/19600 [==============================] - 7s 374us/sample - loss: 0.0622 - acc: 0.9801 - val_loss: 0.3210 - val_acc: 0.8867\n",
      "Epoch 10/10\n",
      "19600/19600 [==============================] - 7s 369us/sample - loss: 0.0538 - acc: 0.9829 - val_loss: 0.3414 - val_acc: 0.8833\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x6406508780>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['acc'])\n",
    " \n",
    "model.fit(train_x, y_label, batch_size=128, epochs=10, validation_data=(x_val, y_val),callbacks=[checkpoints])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.models import load_model\n",
    "model = load_model(os.path.join(save_dir,'model_03-0.8976.hdf5'))\n",
    "loss,acc = model.evaluate(x_val, y_val,batch_size=128,verbose=0)\n",
    "\n",
    "prediction = model.predict(X_test)\n",
    "np.savetxt('submission_model_03-0.8976.txt',prediction, fmt='%.9f')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 改变权重初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.initializers import glorot_normal\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Dense, Dropout, Activation\n",
    "from tensorflow.keras.models import Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_7\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_14 (Dense)             (None, 128)               3416704   \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 3,416,833\n",
      "Trainable params: 3,416,833\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "Train on 19600 samples, validate on 4900 samples\n",
      "Epoch 1/10\n",
      "19600/19600 [==============================] - 8s 406us/sample - loss: 0.4716 - acc: 0.8440 - val_loss: 0.3270 - val_acc: 0.8827\n",
      "Epoch 2/10\n",
      "19600/19600 [==============================] - 7s 373us/sample - loss: 0.2444 - acc: 0.9134 - val_loss: 0.2543 - val_acc: 0.8980\n",
      "Epoch 3/10\n",
      "19600/19600 [==============================] - 7s 368us/sample - loss: 0.1749 - acc: 0.9361 - val_loss: 0.2449 - val_acc: 0.8978\n",
      "Epoch 4/10\n",
      "19600/19600 [==============================] - 7s 370us/sample - loss: 0.1416 - acc: 0.9497 - val_loss: 0.2493 - val_acc: 0.8953\n",
      "Epoch 5/10\n",
      "19600/19600 [==============================] - 7s 371us/sample - loss: 0.1168 - acc: 0.9596 - val_loss: 0.2555 - val_acc: 0.8939\n",
      "Epoch 6/10\n",
      "19600/19600 [==============================] - 7s 368us/sample - loss: 0.0990 - acc: 0.9665 - val_loss: 0.2696 - val_acc: 0.8914\n",
      "Epoch 7/10\n",
      "19600/19600 [==============================] - 7s 372us/sample - loss: 0.0849 - acc: 0.9716 - val_loss: 0.2829 - val_acc: 0.8902\n",
      "Epoch 8/10\n",
      "19600/19600 [==============================] - 7s 369us/sample - loss: 0.0730 - acc: 0.9760 - val_loss: 0.3021 - val_acc: 0.8894\n",
      "Epoch 9/10\n",
      "19600/19600 [==============================] - 7s 372us/sample - loss: 0.0618 - acc: 0.9800 - val_loss: 0.3199 - val_acc: 0.8835\n",
      "Epoch 10/10\n",
      "19600/19600 [==============================] - 7s 368us/sample - loss: 0.0535 - acc: 0.9826 - val_loss: 0.3404 - val_acc: 0.8853\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x640a464f28>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "save_dir = r'D:\\v-yanx\\masijia\\text_mood_classification\\text_mood_classification\\weights_check'\n",
    "weight_path = 'model_{epoch:02d}-{val_acc:.4f}_glorot_normal.hdf5'\n",
    "checkpoints = ModelCheckpoint(os.path.join(save_dir,weight_path), monitor='val_acc', verbose=0, save_best_only=False, save_weights_only=False, mode='max')\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(128, input_shape=(26692,), activation='relu', kernel_initializer=glorot_normal(seed=None)))\n",
    "model.add(Dropout(0.4))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "model.summary()\n",
    "model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['acc'])\n",
    "model.fit(train_x, y_label, batch_size=128, epochs=10, validation_data=(x_val, y_val),callbacks=[checkpoints])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n"
     ]
    }
   ],
   "source": [
    "\n",
    "model = load_model(os.path.join(save_dir,'model_02-0.8980_glorot_normal.hdf5'))\n",
    "loss,acc = model.evaluate(x_val, y_val,batch_size=128,verbose=0)\n",
    "\n",
    "prediction = model.predict(X_test)\n",
    "np.savetxt('submission_model_02-0.8980_glorot_normal.txt',prediction, fmt='%.9f')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MLP512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "Train on 19600 samples, validate on 4900 samples\n",
      "Epoch 1/10\n",
      "19600/19600 [==============================] - 10s 493us/sample - loss: 0.3594 - acc: 0.8661 - val_loss: 0.2562 - val_acc: 0.8994\n",
      "Epoch 2/10\n",
      "19600/19600 [==============================] - 9s 448us/sample - loss: 0.1654 - acc: 0.9380 - val_loss: 0.2540 - val_acc: 0.8961\n",
      "Epoch 3/10\n",
      "19600/19600 [==============================] - 9s 448us/sample - loss: 0.1121 - acc: 0.9603 - val_loss: 0.2805 - val_acc: 0.8927\n",
      "Epoch 4/10\n",
      "19600/19600 [==============================] - 9s 450us/sample - loss: 0.0818 - acc: 0.9713 - val_loss: 0.3130 - val_acc: 0.8918\n",
      "Epoch 5/10\n",
      "19600/19600 [==============================] - 9s 453us/sample - loss: 0.0601 - acc: 0.9791 - val_loss: 0.3501 - val_acc: 0.8880\n",
      "Epoch 6/10\n",
      "19600/19600 [==============================] - 9s 455us/sample - loss: 0.0429 - acc: 0.9866 - val_loss: 0.3981 - val_acc: 0.8822\n",
      "Epoch 7/10\n",
      "19600/19600 [==============================] - 9s 472us/sample - loss: 0.0305 - acc: 0.9898 - val_loss: 0.4447 - val_acc: 0.8827\n",
      "Epoch 8/10\n",
      "19600/19600 [==============================] - 9s 477us/sample - loss: 0.0207 - acc: 0.9932 - val_loss: 0.4995 - val_acc: 0.8786\n",
      "Epoch 9/10\n",
      "19600/19600 [==============================] - 10s 502us/sample - loss: 0.0137 - acc: 0.9954 - val_loss: 0.5623 - val_acc: 0.8747\n",
      "Epoch 10/10\n",
      "19600/19600 [==============================] - 10s 514us/sample - loss: 0.0090 - acc: 0.9972 - val_loss: 0.6226 - val_acc: 0.8716\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x26bcc98cc0>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "save_dir = r'D:\\v-yanx\\masijia\\text_mood_classification\\text_mood_classification\\weights_check\\1'\n",
    "weight_path = 'model_{epoch:02d}-{val_acc:.4f}.hdf5'\n",
    "checkpoints = ModelCheckpoint(os.path.join(save_dir,weight_path), monitor='val_acc', verbose=0, save_best_only=False, save_weights_only=False, mode='max')\n",
    "# Multilayer Perceptron(MLP)\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from tensorflow.keras.models import Sequential\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(512, input_shape=(26692,), activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['acc'])\n",
    " \n",
    "model.fit(train_x, y_label, batch_size=128, epochs=10, validation_data=(x_val, y_val),callbacks=[checkpoints])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.models import load_model\n",
    "model = load_model(os.path.join(save_dir,'model_01-0.8994.hdf5'))\n",
    "loss,acc = model.evaluate(x_val, y_val,batch_size=128,verbose=0)\n",
    "\n",
    "prediction = model.predict(X_test)\n",
    "np.savetxt(r'submit\\tf-idf\\submission_MLP512_01-0.8994.txt',prediction, fmt='%.9f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_2 (Dense)              (None, 512)               13666816  \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1)                 513       \n",
      "=================================================================\n",
      "Total params: 13,667,329\n",
      "Trainable params: 13,667,329\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "Train on 19600 samples, validate on 4900 samples\n",
      "Epoch 1/10\n",
      "19600/19600 [==============================] - 10s 496us/sample - loss: 0.3566 - acc: 0.8671 - val_loss: 0.2563 - val_acc: 0.8957\n",
      "Epoch 2/10\n",
      "19600/19600 [==============================] - 9s 454us/sample - loss: 0.1633 - acc: 0.9381 - val_loss: 0.2893 - val_acc: 0.8859\n",
      "Epoch 3/10\n",
      "19600/19600 [==============================] - 9s 454us/sample - loss: 0.1116 - acc: 0.9611 - val_loss: 0.2905 - val_acc: 0.8894\n",
      "Epoch 4/10\n",
      "19600/19600 [==============================] - 9s 457us/sample - loss: 0.0795 - acc: 0.9729 - val_loss: 0.3162 - val_acc: 0.8871\n",
      "Epoch 5/10\n",
      "19600/19600 [==============================] - 9s 459us/sample - loss: 0.0582 - acc: 0.9804 - val_loss: 0.3569 - val_acc: 0.8835\n",
      "Epoch 6/10\n",
      "19600/19600 [==============================] - 9s 455us/sample - loss: 0.0417 - acc: 0.9864 - val_loss: 0.4011 - val_acc: 0.8808\n",
      "Epoch 7/10\n",
      "19600/19600 [==============================] - 9s 456us/sample - loss: 0.0286 - acc: 0.9907 - val_loss: 0.4602 - val_acc: 0.8810\n",
      "Epoch 8/10\n",
      "19600/19600 [==============================] - 9s 460us/sample - loss: 0.0199 - acc: 0.9937 - val_loss: 0.5131 - val_acc: 0.8792\n",
      "Epoch 9/10\n",
      "19600/19600 [==============================] - 9s 460us/sample - loss: 0.0132 - acc: 0.9956 - val_loss: 0.5673 - val_acc: 0.8743\n",
      "Epoch 10/10\n",
      "19600/19600 [==============================] - 9s 458us/sample - loss: 0.0084 - acc: 0.9973 - val_loss: 0.6369 - val_acc: 0.8743\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x29fa844588>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "save_dir = r'D:\\v-yanx\\masijia\\text_mood_classification\\text_mood_classification\\weights_check\\2'\n",
    "weight_path = 'model_{epoch:02d}-{val_acc:.4f}_glorot_normal.hdf5'\n",
    "checkpoints = ModelCheckpoint(os.path.join(save_dir,weight_path), monitor='val_acc', verbose=0, save_best_only=False, save_weights_only=False, mode='max')\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(512, input_shape=(26692,), activation='relu', kernel_initializer=glorot_normal(seed=None)))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "model.summary()\n",
    "model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['acc'])\n",
    "model.fit(train_x, y_label, batch_size=128, epochs=10, validation_data=(x_val, y_val),callbacks=[checkpoints])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n"
     ]
    }
   ],
   "source": [
    "model = load_model(os.path.join(save_dir,'model_01-0.8957_glorot_normal.hdf5'))\n",
    "loss,acc = model.evaluate(x_val, y_val,batch_size=128,verbose=0)\n",
    "\n",
    "prediction = model.predict(X_test)\n",
    "np.savetxt(r'submit\\tf-idf\\submission_MLP512_01-0.8957_glorot_normal.txt',prediction, fmt='%.9f')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MLP640"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_4 (Dense)              (None, 640)               17083520  \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 640)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 1)                 641       \n",
      "=================================================================\n",
      "Total params: 17,084,161\n",
      "Trainable params: 17,084,161\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "Train on 19600 samples, validate on 4900 samples\n",
      "Epoch 1/10\n",
      "19600/19600 [==============================] - 10s 519us/sample - loss: 0.3469 - acc: 0.8684 - val_loss: 0.2569 - val_acc: 0.8998\n",
      "Epoch 2/10\n",
      "19600/19600 [==============================] - 9s 484us/sample - loss: 0.1594 - acc: 0.9400 - val_loss: 0.2592 - val_acc: 0.9000\n",
      "Epoch 3/10\n",
      "19600/19600 [==============================] - 10s 485us/sample - loss: 0.1066 - acc: 0.9619 - val_loss: 0.3091 - val_acc: 0.8896\n",
      "Epoch 4/10\n",
      "19600/19600 [==============================] - 10s 485us/sample - loss: 0.0754 - acc: 0.9751 - val_loss: 0.3382 - val_acc: 0.8869\n",
      "Epoch 5/10\n",
      "19600/19600 [==============================] - 10s 491us/sample - loss: 0.0537 - acc: 0.9821 - val_loss: 0.3715 - val_acc: 0.8849\n",
      "Epoch 6/10\n",
      "19600/19600 [==============================] - 10s 493us/sample - loss: 0.0373 - acc: 0.9884 - val_loss: 0.4149 - val_acc: 0.8812\n",
      "Epoch 7/10\n",
      "19600/19600 [==============================] - 10s 486us/sample - loss: 0.0247 - acc: 0.9920 - val_loss: 0.4836 - val_acc: 0.8780\n",
      "Epoch 8/10\n",
      "19600/19600 [==============================] - 10s 491us/sample - loss: 0.0153 - acc: 0.9955 - val_loss: 0.5611 - val_acc: 0.8765\n",
      "Epoch 9/10\n",
      "19600/19600 [==============================] - 10s 496us/sample - loss: 0.0099 - acc: 0.9970 - val_loss: 0.6212 - val_acc: 0.8716\n",
      "Epoch 10/10\n",
      "19600/19600 [==============================] - 10s 493us/sample - loss: 0.0059 - acc: 0.9983 - val_loss: 0.6817 - val_acc: 0.8708\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x29fb5d45f8>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "save_dir = r'D:\\v-yanx\\masijia\\text_mood_classification\\text_mood_classification\\weights_check\\3'\n",
    "weight_path = 'model_{epoch:02d}-{val_acc:.4f}.hdf5'\n",
    "checkpoints = ModelCheckpoint(os.path.join(save_dir,weight_path), monitor='val_acc', verbose=0, save_best_only=False, save_weights_only=False, mode='max')\n",
    "# Multilayer Perceptron(MLP)\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from tensorflow.keras.models import Sequential\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(640, input_shape=(26692,), activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['acc'])\n",
    " \n",
    "model.fit(train_x, y_label, batch_size=128, epochs=10, validation_data=(x_val, y_val),callbacks=[checkpoints])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n"
     ]
    }
   ],
   "source": [
    "model = load_model(os.path.join(save_dir,'model_02-0.9000.hdf5'))\n",
    "loss,acc = model.evaluate(x_val, y_val,batch_size=128,verbose=0)\n",
    "\n",
    "prediction = model.predict(X_test)\n",
    "np.savetxt(r'submit\\tf-idf\\submission_MLP640_02-0.9000.txt',prediction, fmt='%.9f')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MLP256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_8 (Dense)              (None, 256)               6833408   \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 6,833,665\n",
      "Trainable params: 6,833,665\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "Train on 19600 samples, validate on 4900 samples\n",
      "Epoch 1/10\n",
      "19600/19600 [==============================] - 9s 435us/sample - loss: 0.3955 - acc: 0.8582 - val_loss: 0.2726 - val_acc: 0.8918\n",
      "Epoch 2/10\n",
      "19600/19600 [==============================] - 8s 403us/sample - loss: 0.1862 - acc: 0.9315 - val_loss: 0.2521 - val_acc: 0.8994\n",
      "Epoch 3/10\n",
      "19600/19600 [==============================] - 8s 403us/sample - loss: 0.1291 - acc: 0.9538 - val_loss: 0.2612 - val_acc: 0.8998\n",
      "Epoch 4/10\n",
      "19600/19600 [==============================] - 8s 403us/sample - loss: 0.0968 - acc: 0.9668 - val_loss: 0.2867 - val_acc: 0.8939\n",
      "Epoch 5/10\n",
      "19600/19600 [==============================] - 8s 405us/sample - loss: 0.0740 - acc: 0.9756 - val_loss: 0.3126 - val_acc: 0.8871\n",
      "Epoch 6/10\n",
      "19600/19600 [==============================] - 8s 404us/sample - loss: 0.0575 - acc: 0.9809 - val_loss: 0.3414 - val_acc: 0.8853\n",
      "Epoch 7/10\n",
      "19600/19600 [==============================] - 8s 401us/sample - loss: 0.0444 - acc: 0.9857 - val_loss: 0.3782 - val_acc: 0.8835\n",
      "Epoch 8/10\n",
      "19600/19600 [==============================] - 8s 407us/sample - loss: 0.0348 - acc: 0.9883 - val_loss: 0.4194 - val_acc: 0.8802\n",
      "Epoch 9/10\n",
      "19600/19600 [==============================] - 8s 407us/sample - loss: 0.0259 - acc: 0.9921 - val_loss: 0.4514 - val_acc: 0.8776\n",
      "Epoch 10/10\n",
      "19600/19600 [==============================] - 8s 407us/sample - loss: 0.0193 - acc: 0.9942 - val_loss: 0.5020 - val_acc: 0.8759\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x29f92c2d68>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "save_dir = r'D:\\v-yanx\\masijia\\text_mood_classification\\text_mood_classification\\weights_check\\4'\n",
    "weight_path = 'model_{epoch:02d}-{val_acc:.4f}.hdf5'\n",
    "checkpoints = ModelCheckpoint(os.path.join(save_dir,weight_path), monitor='val_acc', verbose=0, save_best_only=False, save_weights_only=False, mode='max')\n",
    "# Multilayer Perceptron(MLP)\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from tensorflow.keras.models import Sequential\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(256, input_shape=(26692,), activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['acc'])\n",
    " \n",
    "model.fit(train_x, y_label, batch_size=128, epochs=10, validation_data=(x_val, y_val),callbacks=[checkpoints])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n",
      "WARNING:tensorflow:Falling back from v2 loop because of error: Failed to find data adapter that can handle input: <class 'scipy.sparse.csr.csr_matrix'>, <class 'NoneType'>\n"
     ]
    }
   ],
   "source": [
    "model = load_model(os.path.join(save_dir,'model_03-0.8998.hdf5'))\n",
    "loss,acc = model.evaluate(x_val, y_val,batch_size=128,verbose=0)\n",
    "\n",
    "prediction = model.predict(X_test)\n",
    "np.savetxt(r'submit\\tf-idf\\submission_MLP256_03-0.8998.txt',prediction, fmt='%.9f')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8546938775510204\n",
      "0.9254591836734694\n",
      "[[2083  346]\n",
      " [ 366 2105]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB  \n",
    "from sklearn import metrics\n",
    "bys = MultinomialNB(alpha = 0.01)   \n",
    "bys.fit(train_x, y_label);  \n",
    "\n",
    "y_pred = bys.predict(x_val)\n",
    "acc = bys.score(x_val, y_val)\n",
    "print(acc)\n",
    "acc = bys.score(train_x, y_label)\n",
    "print(acc)\n",
    "confusion_array_bys = confusion_matrix(y_val, y_pred)\n",
    "print(confusion_array_bys)\n",
    "#        pred_0 pred_1\n",
    "#act_0   2083    346\n",
    "#act_1   366     2105\n",
    "y_pred_prob = bys.predict_proba(X_test) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "bys_list = []\n",
    "bys_val = []\n",
    "bys_train = []\n",
    "bys_auc = []\n",
    "for i in np.arange(0.01,4.02,0.02):\n",
    "        bys = MultinomialNB(alpha = i)\n",
    "        bys.fit(train_x,y_label) \n",
    "        y_pred = bys.predict(x_val)\n",
    "        val_acc =  bys.score(x_val, y_val)\n",
    "        auc = roc_auc_score(y_val, y_pred)\n",
    "        bys_list.append(bys)\n",
    "        bys_val.append(val_acc)\n",
    "        bys_auc.append(auc)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bys_auc.index(0.8645778057163382)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8644897959183674\n",
      "0.907704081632653\n",
      "[[2125  304]\n",
      " [ 360 2111]]\n"
     ]
    }
   ],
   "source": [
    "bys = bys_list[25]   \n",
    "bys.fit(train_x, y_label);  \n",
    "\n",
    "y_pred = bys.predict(x_val)\n",
    "acc = bys.score(x_val, y_val)\n",
    "print(acc)\n",
    "acc = bys.score(train_x, y_label)\n",
    "print(acc)\n",
    "confusion_array_bys = confusion_matrix(y_val, y_pred)\n",
    "print(confusion_array_bys)\n",
    "#        pred_0 pred_1\n",
    "#act_0   2125     304\n",
    "#act_1   360     2111\n",
    "y_pred_prob = bys.predict_proba(X_test) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=0.5099999999999999, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bys_list[25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['submit\\\\tf-idf\\\\train_MNB_alpha0.51.m']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.savetxt(r'submit\\tf-idf\\submission_MNB_alpha0.51.txt',y_pred_prob[:,1], fmt='%.9f')\n",
    "joblib.dump(bys_list[25], r'submit\\tf-idf\\train_MNB_alpha0.51.m')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\v-yanx\\anaconda3\\envs\\captcha\\lib\\site-packages\\sklearn\\ensemble\\forest.py:245: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7412244897959184\n",
      "0.9934183673469388\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn import model_selection\n",
    "# random forest model creation\n",
    "rfc0 = RandomForestClassifier()\n",
    "rfc0.fit(train_x, y_label)\n",
    "y_pred = rfc0.predict(x_val)\n",
    "acc = rfc0.score(x_val, y_val)\n",
    "print(acc)\n",
    "acc = rfc0.score(train_x, y_label)\n",
    "print(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "                       max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=10,\n",
       "                       n_jobs=None, oob_score=False, random_state=None,\n",
       "                       verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfc0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 先粗略估计每个参数变化所引起的准确率变化方向，再进行总体for循环调参\n",
    "1.min_samples_split=10:  \n",
    "\n",
    "max_depth 30\\40\\50 变化 xiao  \n",
    "max_leaf_nodes 5\\10\\15  (max_depth = 30,n_es = 200)\n",
    " 0.8057\\0.8139\\0.8185  \n",
    "n_estimater yaoduo(max_leaf_nodes = 15,max_depth = 30)     200\\300\\350  \n",
    " 0.8185\\0.8230\\0.8235 zaida meishayong\n",
    " 此时改变max_depth=20，变化也不大.danbuneng xiao\n",
    "\n",
    "2.不要增大min_samples_leaf\n",
    "3.(bootstrap=True, class_weight=None, criterion='gini',\n",
    "                       max_depth=30, max_features='auto', max_leaf_nodes=15,\n",
    "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "                       min_samples_leaf=1, min_samples_split=10,\n",
    "                       min_weight_fraction_leaf=0.0, n_estimators=350,\n",
    "                       n_jobs=None, oob_score=True, random_state=0,\n",
    "                       verbose=0, warm_start=False)\n",
    " 增大max_leaf_nodes:20\\25\\40\\50\\60\\100\\150  \n",
    " 0.8243\\0.8267\\0.8292\\0.8302\\0.8312\\0.8351\\0.837959\n",
    "  min_samples_split zuihao bu da =15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.843469387755102\n",
      "0.843469387755102\n",
      "0.8436734693877551\n",
      "0.8436734693877551\n",
      "0.8436734693877551\n",
      "0.8436734693877551\n",
      "0.8436734693877551\n",
      "0.8436734693877551\n",
      "0.8436734693877551\n",
      "0.8436734693877551\n"
     ]
    }
   ],
   "source": [
    "val_acc=[]\n",
    "rfc_list = []\n",
    "for i in range(1000,2000,100):\n",
    "    rfc = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
    "                       max_depth=30, max_features='auto', max_leaf_nodes=i,\n",
    "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "                       min_samples_leaf=1, min_samples_split=15,\n",
    "                       min_weight_fraction_leaf=0.0, n_estimators=350,\n",
    "                       n_jobs=None, oob_score=True, random_state=0,\n",
    "                       verbose=0, warm_start=False)\n",
    "    rfc.fit(train_x, y_label)\n",
    "    y_pred = rfc.predict(x_val)\n",
    "    val_acc.append(rfc.score(x_val, y_val))\n",
    "    rfc_list.append(rfc)\n",
    "    acc = rfc.score(x_val, y_val)\n",
    "    print(acc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}