text_preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data preprocessing and look up how many words are there in every sentances\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import csv\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pandas import DataFrame\n",
    "path = r'D:\\v-yanx\\masijia\\text_mood_classification\\text_mood_classification\\public_data\\labeledTrainData.tsv'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_un = r'D:\\v-yanx\\masijia\\text_mood_classification\\text_mood_classification\\public_data\\unlabeledTrainData.tsv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentiments = []\n",
    "sentances = []\n",
    "with open(path, encoding=\"utf8\") as tsvfile:\n",
    "    data = csv.reader(tsvfile, delimiter=\"\\t\")\n",
    "    for line in data:\n",
    "        sentiments.append(line[1])\n",
    "        sentances.append(line[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24501"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sentances)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = sentances[1]\n",
    "text = ''.join([c if c.isalpha() else ' ' for c in t])# clear ,#*&!@$...., only leave alphas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'This has to be the funniest stand up comedy I have ever seen  Eddie Izzard is a genius  he picks in Brits  Americans and everyone in between  His style is completely natural and completely hilarious  I doubt that anyone could sit through this and not laugh their a   off  Watch  enjoy  it s funny '"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "word = [w for w in text.split(' ') if w]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_dic = {}\n",
    "for i in range(1,len(sentances)):\n",
    "    text = ''.join([c if c.isalpha() else ' ' for c in sentances[i]])\n",
    "    words = [w for w in text.split(' ') if w]\n",
    "    key = '{:0>5}_{}'.format(i,sentiments[i])\n",
    "    words_dic[key] = words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('words_dic.npy',words_dic)# save the sentances and their ids in a dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_dic_train = np.load('words_dic.npy',allow_pickle=True).item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "lens = []\n",
    "for key in words_dic_train:\n",
    "    l = len(words_dic_train[key])\n",
    "    lens.append(l)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "uni = set(lens)\n",
    "x = []\n",
    "y = []\n",
    "for i in uni:\n",
    "    x.append(i)\n",
    "x.sort()\n",
    "#y.append(lens.count(i))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in x:\n",
    "    y.append(lens.count(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAesklEQVR4nO3de3RcZb3/8fc3SS/0Si9pKb2QUiqCUgpEBBEooEIBBRQRjkvrEq3+Dl3LI7p+gshPjkeEg6DHIwqCVopLC5wDKMqtUK5CoaTQll5oaUtLr2looffmMvP9/TE76bSdSSYzszOZvT+vtbKy59l7Mt+dgU93nnn285i7IyIi0VJR6gJERKT4FO4iIhGkcBcRiSCFu4hIBCncRUQiqKrUBQAMHTrUa2pqSl2GiEhZmTdv3nvuXp1pX7cI95qaGurq6kpdhohIWTGzNdn2qVtGRCSCFO4iIhGkcBcRiSCFu4hIBCncRUQiSOEuIhJBCncRkQiKbbjvbGzhr2+sL3UZIiKh6BY3MZXCDx96k0cWbGBcdT+OGzWw1OWIiBRVbK/cN23fC8CuppYSVyIiUnyxDfcde1OhntRKVCISQbEN96UbtwOgbBeRKIptuLfSlbuIRFGH4W5m081ss5ktSmu738zmB1+rzWx+0F5jZnvS9t0ZZvHFkFS2i0gE5TJa5h7gduDe1gZ3/1LrtpndBmxLO36lu08sVoFhm/nqu5z5oYzTIYuIlK0Ow93dXzCzmkz7zMyAy4Czi1tWuDYHI2UAnli8qYSViIiEo9A+99OBend/O61trJm9YWbPm9np2Z5oZlPNrM7M6hoaGgoso3P+Ol83L4lItBUa7lcAM9MebwTGuPsJwNXAX8xsQKYnuvtd7l7r7rXV1V3bLVJh1qWvJyLS1fIOdzOrAj4P3N/a5u6N7r4l2J4HrAQ+VGiRxaZwF5GoK+TK/VPAW+6+rrXBzKrNrDLYPhIYD6wqrMTiq6xQuItItOUyFHImMAc42szWmdmVwa7L2b9LBuAMYKGZLQD+F/i2u28tZsHFUKFwF5GIy2W0zBVZ2r+Woe1B4MHCywqXsl1Eoi6Wd6hWqs9dRCIuluGuD1RFJOriGe7qlxGRiItnuCvbRSTiYhnuGgopIlEXy3BXn7uIRF0sw11X7iISdbEMd125i0jUxTLcle0iEnWxDHetrCciURfTcFe6i0i0xTPcS12AiEjIYhnuSV25i0jExTLcle0iEnWxDHdduYtI1MUy3EVEoi6W4f6d++aXugQRkVDFMtxFRKIulzVUp5vZZjNblNZ2g5mtN7P5wdf5afuuNbMVZrbMzM4Nq/Bi0rh3EYmaXK7c7wHOy9D+S3efGHw9BmBmx5JaOPsjwXN+a2aVxSq22C49aRSg0TMiEj0dhru7vwBszfHnXQTc5+6N7v4OsAI4uYD6QlUVzA6p0TMiEjWF9LlPM7OFQbfNoKBtJLA27Zh1QdtBzGyqmdWZWV1DQ0MBZeSvoi3cS/LyIiKhyTfc7wDGAROBjcBtQXum+RYzRqe73+Xute5eW11dnWcZhdGVu4hEVV7h7u717p5w9yRwN/u6XtYBo9MOHQVsKKzE8FQq3EUkovIKdzMbkfbwEqB1JM0jwOVm1svMxgLjgbmFlVhc6SNjKk3dMiISTVUdHWBmM4FJwFAzWwf8GJhkZhNJdbmsBr4F4O6LzewBYAnQAlzl7olwSs9P+kV6ZaWu3EUkmjoMd3e/IkPzH9o5/kbgxkKKClN6jLdeuU+4YRaL/v1c+vXq8NchIlIWYneHanq3TFXaQtlbdzaVohwRkVDELtyz9a/f8/LqLq1DRCRMsQt3T+uY2duSbNue/tI7pShHRCQU8Qv3tCv3Css0LF9EpPzFOtxdq6mKSETFL9zZL91FRCIpduGeVLaLSAzELtw1d7uIxEH8wj19W0EvIhEVv3DfN/qRRDL7cSIi5Sx+4Z527a6RkCISVfEL97SemB6VsTt9EYmJ2KVb+gyQ6nMXkaiKXbinx7mm+hWRqIpduLcG+hdOHKVFOkQksmIX7q2X7ieMOfSgK/e1W3eXoCARkeKLXbi3xnmFGQf2ykz7y+tdXo+ISBhiF+6tV+tm8LVP1Oy3r0X9NCISER2Gu5lNN7PNZrYore3nZvaWmS00s4fN7NCgvcbM9pjZ/ODrzjCLz0fr1boBNUP7ZtwnIlLucrlyvwc474C2p4CPuvsEYDlwbdq+le4+Mfj6dnHKLJ70bpls+0REyl2H4e7uLwBbD2ib5e4twcNXgFEh1BaKZDLt0l1EJKKK0ef+deDxtMdjzewNM3vezE7P9iQzm2pmdWZW19DQUIQyOidTtje2JLq8DhGRMBQU7mZ2HdAC/Dlo2giMcfcTgKuBv5jZgEzPdfe73L3W3Wurq6sLKaNTWvvVM3XLrGrY1WV1iIiEKe9wN7MpwIXAlz24j9/dG919S7A9D1gJfKgYhRZL+mgZEZGoyivczew84AfA59x9d1p7tZlVBttHAuOBVcUotFja+0BVRCQqqjo6wMxmApOAoWa2DvgxqdExvYCnLBWSrwQjY84AfmJmLUAC+La7b834g0tEV+4iEgcdhru7X5Gh+Q9Zjn0QeLDQosKksewiEgexu0O1tWNG3TIiEmWxC/e2Ye7KdhGJsNiF+77pB1LpftPnjythNSIi4YhfuLd1y6QeX3HyGC6cMKKEFYmIFF/swn1PU+ou1PRuGX3GKiJRE7tw/97/LABg+56WjPuXbdrRleWIiIQiduHeOsXAfvPIpF26N+xo7OKKRESKL3bh3sqyDJfRotkiEgWxDff0ce6uXncRiZgYh/u+7fSL9Z89trTrixERKbIYh3valXtauL+lD1RFJAJiG+66Q1VEoiy24V5ZoT53EYmu2IZ7Og2QEZGoUbgD084+qtQliIgUVWzDPb3PfcKoQ9u2B/XpUYJqRESKK7bhns3Rh/UvdQkiIgWLbbi3Tvl7IPW/i0gUxDbcs42QUbiLSBR0GO5mNt3MNpvZorS2wWb2lJm9HXwflLbvWjNbYWbLzOzcsAoPi4ZFikgU5HLlfg9w3gFt1wCz3X08MDt4jJkdC1wOfCR4zm/NrLJo1RZRtm6ZpLJdRCKgw3B39xeArQc0XwTMCLZnABentd/n7o3u/g6wAji5SLV2iYTSXUQiIN8+9+HuvhEg+D4saB8JrE07bl3QdhAzm2pmdWZW19DQkGcZxadwF5EoKPYHqpn6OjKmpbvf5e617l5bXV1d5DI6lm1umTfXb2PR+m1dW4yISJHlG+71ZjYCIPi+OWhfB4xOO24UsCH/8sLT3qiYS+98uesKEREJQb7h/ggwJdieAvwtrf1yM+tlZmOB8cDcwkrsenubk6UuQUSkIFUdHWBmM4FJwFAzWwf8GLgZeMDMrgTeBb4I4O6LzewBYAnQAlzl7omMP7jENOWviERZh+Hu7ldk2XVOluNvBG4spKjuYMXmHRw1TFMRiEh5iu0dqh351C9eKHUJIiJ5U7iLiESQwl1EJIJiG+7DB/Tu8Jhn3qrvgkpERIovduF+8cTDATjlyCEdHvvsW93nzlkRkc6IXbibGWMG9zmo/cX/exb/++1TS1CRiEjxdTgUMmqS7hnHuI8e3IfRGUJfRKQcxerKfW9zgr/N38B7OxpzOv7+19Z2fJCISDcUq3CftST1Aemuptxumm1KaBoCESlPsQr3Ck05ICIxEatwz7b6kohI1MQq3HXlLiJxEatwN00FKSIxEatw15W7iMRFrMI9nyt3b2/JJhGRbipW4Z7Plft5//Vi8QsREQlZrMK9Mo90X1a/I4RKRETCFatw79MzNdvCTZ8/rsSViIiEK++5ZczsaOD+tKYjgf8HHAp8E2idUvGH7v5Y3hUWUTLoPz9iSPY5ZHr3qNAC2SJS9vK+cnf3Ze4+0d0nAicBu4GHg92/bN3XXYIdYGXDzg6POSyHed5FRLq7YnXLnAOsdPc1Rfp5obju4UUArGrYlfWYTCNqVuXwj4KISHdSrHC/HJiZ9niamS00s+lmNijTE8xsqpnVmVldQ0P3WRTjlksnHNR22e/mlKASEZH8FRzuZtYT+BzwP0HTHcA4YCKwEbgt0/Pc/S53r3X32urq6kLLKJqP1QxmQO/9P4rYvrelRNWIiOSnGFfuk4HX3b0ewN3r3T3h7kngbuDkIrxGl6o4YMhkU4s+YBWR8lKMcL+CtC4ZMxuRtu8SYFERXqNLVWTod396iRbLFpHyUdAye2bWB/g08K205lvMbCLgwOoD9pVUzZA+rN6ym2NG9G/3uEz3Om3YtiekqkREiq+gcHf33cCQA9q+UlBFIbpwwuHc/uwKTjpicLvHafZIESl3sbpDNeFOz8qOT/mjhw84qO32Z1aEUZKISCjiFe5Jz2l+mV//y4kHtW3OcVFtEZHuIFbh3pLILdz79Sqot0pEpORiFe6JZDKvmSFFRMpNvMLdnSqFu4jEQLzCPcc+dxGRchercM+1zz2b5kSSGS+vZldjC3+as5pkUkvwiUj3FKtPDhNeWLj/6um3uf3ZFdz46FKaEkkO7dOTzx5/eBErFBEpjlhduSeSufe5f+OTYw9qu/3Z1Fj3pkRqrpndTZpQTES6p8iH+6Zte9nTlACgJekHTQqWzY8uPJbDB7a/cIeh/nsR6Z4iH+6n3DSbK+5+BXfng91NnRot8+IPzg6xMhGR8MSiz33+2g+48/lVvLRiC4P79sz5eRpZIyLlKvJX7q2eXLwJgK27mkpciYhI+GIT7vPXflDqEkREukxku2WueXAhvapC/rdLvTYi0k1F8sp9y85G7nttLTPmrCn4Z93w2WOLUJGISNeKZLj/6ZXsod7Zz0iH9u/V7v4XljfwpzmrO/dDRURCFslumaUbt2fdl2l91PZ4OzMMGPDV6XMB+MqpNZ36uSIiYSp0DdXVwA4gAbS4e62ZDQbuB2pIraF6mbu/X1iZnfPk4uyLWXc23EVEylExumXOcveJ7l4bPL4GmO3u44HZweNuQ2PXRSQOwuhzvwiYEWzPAC4O4TXy1tn53CeOPjTrvl2NmltGRLqnQsPdgVlmNs/MpgZtw919I0DwfVimJ5rZVDOrM7O6hoaGAsvIXWcn6R09uE/WfTf8fUlhxYiIhKTQcD/N3U8EJgNXmdkZuT7R3e9y91p3r62uri6wjNztzONq+/oLNRxSRMpLQeHu7huC75uBh4GTgXozGwEQfN9caJGl1rtHJEeMikiE5Z1aZtbXzPq3bgOfARYBjwBTgsOmAH8rtMjOuP+1d9vd/+HD+nf6Zw7r3/7UvyIi3U0hQyGHAw9bamhhFfAXd3/CzF4DHjCzK4F3gS8WXmbufv/iO+3uH9Qn91khW33qmIwfG4iIdFt5h7u7rwKOz9C+BTinkKIK0dFQx4o8/lYxMw4b0JtN2/fmWZWISNeKXGdyRzcp5XsT05PfzfmzYhGRkotUuLs7a7bsavcYyzPcBx7SI6/niYiUQqTCfc2W3ewK1kvNJqwbVL29SWhERLpYpMK9lPPG7GlOMPXeOtZu3V2yGkREWkUq3KsqOw73r582NpTX/s/H32LWknpOv+VZ9nTw14OISNgiFe65XLif8aFw7oat397Ytn3dX98M5TVERHIVqXBP7/bu36trp6pf2bCzbfuF5e/RkkiyaZuGTopIaUQq3BPJfeneq0dll77225v3hft7Oxv58u9f5ZSbZvPB7iYA9jYn9qtPRCRMkQr39Cv3njn0v+fr8e+c3uExr76zFYAde1MTlX34+if42h/nhlaTiEi6SC2zl0xL9x5V+/+79YcptRwxJPv0vZ0xtF/766pm8+Lb7xXl9UVEOhKpK/f0cB9X3W+/feccM5yjhnV+0rBMqvv3YlgHC2eLiJRSxMJ93/YvL5vIrV88aOqbojknx8nEtGSriJRCpMK99S7RKacewcA+Pbj0pFGhvdY1k4/J6bj0IZIiIl0lUuGeCML940cOCf21cp1r5tYnl4VciYjIwSIV7slk6nsppyE4UFJzzohICURytEz65GDfPH0sm3eUrmvEHZ5aUl+y1xeReIpouO9L9+suKO3i1nNXb2Xu6q1tj7ftaWZ5/Q4+VjO4hFWJSNRFqlumqSXVL9OzqmtOK5+Fsy/89Yt88c457G3W5GIiEp5ohXsiFe49KsM7rXwW2E63duseAJqDWkVEwpB3t4yZjQbuBQ4DksBd7v4rM7sB+CbQEBz6Q3d/rNBCc9GcSHXLhHXlvuQn51KVzyKsGbT+lSEiEoZC+txbgO+5++tm1h+YZ2ZPBft+6e63Fl5e57R1y4R05d6n5/6/rkIGwjQq3EUkRHmnoLtvdPfXg+0dwFJgZLEKy0drV0ePqu4zFDKbT9z8DFt26gYnEQlHUS5xzawGOAF4NWiaZmYLzWy6mQ3K8pypZlZnZnUNDQ2ZDum01nAP68r9QCMG9i7o+au3aEk+EQlHwSloZv2AB4F/c/ftwB3AOGAisBG4LdPz3P0ud69199rq6uKsjtTa1RHmB6rpZk49hV9dPnG/thlfPznn51dWGN+YUcdf31hf7NJEJOYKGuduZj1IBfuf3f0hAHevT9t/N/CPgirshOeXp/4C6KqhkCMGHsJFE0ey7v09/DyYZqB3J17717PfZvZbm3l6aT0L122jqtIY1r8X3zj9yLBKFpGYKGS0jAF/AJa6+y/S2ke4+8bg4SXAosJKzN2jC1Mv29XTD/zrpHHMW/M+SXdGDjok5+fNfmtz2/b0l95p21a4i0ihCrlyPw34CvCmmc0P2n4IXGFmEwEHVgPfKqjCPAzu27NLX8/MmP61jwFQv73wdVOXbtzO6vd2Mfm4EQX/LBGJp7zD3d3/CWS6RO6SMe3tqawo3WiZYvzVMPlXLwLw9o2T+fuCDVxywkisG02GJiLdX6TmlukOqtL+YTlqWD9WpC2c3VnfvLeO55Y10LOqggsnHF6M8kQkJiI1/UB3UBkszN27RwVPX31mQT/ruWWpD4hbF9kWEcmVwr3IKoPuk9Yl/9656fyCf+b7u5vY06SJxkQkd5EJ90cWbCh1CcDB/f3F6Cu/5YllTLr1WfY2J2hJJEkmncaWBImkFgIRkcwi0+delzZneim1hvsJow9ta3vu+5PYuruJHXtbmDJ9bl4/t357Ix++/gkAzvvIYTyxeBMnjDmUh//1tMKLFpHIiUy4d5fV7HpUVvD3aZ+kZmiftraaoX2poS8As793Jjv2tnDxb17K+zWeWLwJgDfe/aCwYkUksiLTLdOdHDdqIP17Z15Ae1x1Pw4/tLA5adLdNqv9BbiffWszx1z/BLsa9aGsSJxEJtydbnLpnoNizQkP8OtnVrSt6vT0knqmTJ9LcyLJZXfOYe47W7nlyWXsaU5w8o1Paw55kRiJTLg3NpdPcBX7JqsrZ7zGzsYWvnFvHc8vb+DdrbuZu3orl/1uDks3bgdgV1OCd7fuZs2WXVz70EJatBKUSKRFJtx3l9FQwfQbnSZ/9LD9HufjpRVb+NLv5rQ9Xvf+nozHLdu0gzN//hwz565lwbptBb2miHRvkQn31hGHd3+1trSF5CD9yv375x5dlA6lxRu2t23f+OiSjMdc9ZfXM9YgItETmXD/RzAj5Phh/UpcScd6VlYwdmhffnHZ8Yyr7sctX5iQ8bh8A3h5fcdTHlRmGX+/u6mFfyxM3TOwtznBDY8sZmVD/lMoiEhpRGIo5Lw1+8a4d/V0v/moqDCe/f6ktsdfOGkUXzhpFABn3/ocq97bBcAD3zqVL9zxcig1rGjYwZ7mBL2qKnh80Sb+z5njGNinB//xj6XMnPsu89a8z4K1H/D6ux9wz8ureeXac9ixt5nxw/uHUo+IFFckwn172twrZZDt7Xrm+5OoueZRLv/YaEjrsOnbs5JdRfxc4bv3L9jv8Z3Pr2Tejz7FK6u2APDHl1bvt/+Um2YDsPrmC4BU//3Rh/UnmXRmLdnEOccMb3cFrE3b9nJIj0oG9sk8RDSTLTsbSbgzqE9PVjbsZP37ezh9fHWXLcYiUs4iEe7peV4Rgb7klT87nwqDd7em1li9+tMf4qqzjmLcDw+eTflHFxzDTx9dWpTXPemnT3d4zKL121i0fhvXPPQmP/7sscxZuYVZS+o5enh/nvzuGVmfd8pNsxnQu4qFN5x70L6GHY0M7ddzv6katu1ubqvnggkj2hZiuWDCCH7zLyd29tREYicS4Z6u/KN9X1/7EUP68s8fnMXhAw/J+o/WlE/UFC3cc3Hhr//Ztv3vf9/3we2y+h00J5JUmFFZYSSTqTsP9jYn2q60tx8wu2VzIsmb67fx+d++zM2fP47LTx5DY0uCnpUVHP+TWW3HtQY7pMbyi0jHIhfuUbtRZ9SgPge13fHlE6nu34uh/Xp12WLguRh/3eP0qDSW/3QyV854jWeDKYvTbdy2hxEDD+GD3U1M/MlTbe3PL2/gqGH9uPTOOdQeMSjrazS2JNnbnKB3j8pQzkEkKso+3N2dr/3xtbbHcRjid+Dye09ffSZf+t0ctuxqKlFF+zQnnLHXZl+M69SbngHgM8cO36/dHS69MzVWv27N++2+xoevf4KfXzqBL9aOLrBakegyD2nGLTM7D/gVUAn83t1vznZsbW2t19XV5fU6Ndc8ut/j1g/8oujVVVsYMfAQxgw5+Gp+y85GFqz7gNqawUy4IdWl8d9XnMAZ44e2XSFPO+soelRW8Munl3dp3WH7xWXHc/HEkZH4vEWkM8xsnrtnvLknlCt3M6sEfgN8GlgHvGZmj7h75rtr8rRw3f6zIk4766hi/vhu5+NHDsm6b0i/Xpz94f2vhj93fGppvtnfO5PFG7a3Pf7oyAFcOePgf0xPO2oIL63YUsSKu8bVDyzg1ieX0bdX2f8hKjE06ehqrrvg2KL/3LD+bzgZWOHuqwDM7D7gIqCo4T4gmHlx5KGHsKuphWlnRzvcc/UfF3+U40YObHs8rrof46r33dx1zjHDWf7Tydw2axmf+chhvL7mfb55xpEArNi8g5899hbnHzeCof168tyyBp5f3sD1Fx7D1+/Z9w/ClZ8cy8fHDuaJxZt46PX1QGrN2ONGDuThN9a3HTdh1EC27Gxi/QepKRF6VBrNidRfi2MG96FfryqWBPPfDO3XiyOG9GHemveZdHQ1Kxt2snbrvqkUDhvQm03b97Y9PnbEAC45YSRD+/fk6aWbCeuvUJEwDR9QvFli04XSLWNmlwLnufs3gsdfAT7u7tPSjpkKTAUYM2bMSWvWrCl6HSIiUdZet0xYQy0ydX7u96+Iu9/l7rXuXltdXR1SGSIi8RRWuK8D0ocyjAK6xyKnIiIxEFa4vwaMN7OxZtYTuBx4JKTXEhGRA4Tygaq7t5jZNOBJUkMhp7v74jBeS0REDhba2DF3fwzIfjeLiIiEpvvcuy4iIkWjcBcRiSCFu4hIBIU2t0ynijBrAPK5i2ko8F6Ry+nudM7xELdzjtv5QnHO+Qh3z3ijULcI93yZWV22u7OiSuccD3E757idL4R/zuqWERGJIIW7iEgElXu431XqAkpA5xwPcTvnuJ0vhHzOZd3nLiIimZX7lbuIiGSgcBcRiaCyDXczO8/MlpnZCjO7ptT1FIuZrTazN81svpnVBW2DzewpM3s7+D4o7fhrg9/BMjM7t3SV587MppvZZjNblNbW6XM0s5OC39UKM/tvM+u2i6hmOecbzGx98F7PN7Pz0/aV9Tmb2Wgze9bMlprZYjP7TtAe2fe5nXMuzfvs7mX3RWqmyZXAkUBPYAFwbKnrKtK5rQaGHtB2C3BNsH0N8J/B9rHBufcCxga/k8pSn0MO53gGcCKwqJBzBOYCp5JaHOZxYHKpz62T53wD8P0Mx5b9OQMjgBOD7f7A8uC8Ivs+t3POJXmfy/XKvW2NVndvAlrXaI2qi4AZwfYM4OK09vvcvdHd3wFWkPrddGvu/gKw9YDmTp2jmY0ABrj7HE/933Bv2nO6nSznnE3Zn7O7b3T314PtHcBSYCQRfp/bOedsQj3ncg33kcDatMfraP+XWE4cmGVm84J1ZgGGu/tGSP0HBAwL2qP0e+jsOY4Mtg9sLzfTzGxh0G3T2kURqXM2sxrgBOBVYvI+H3DOUIL3uVzDvcM1WsvYae5+IjAZuMrMzmjn2Cj/HlplO8conPsdwDhgIrARuC1oj8w5m1k/4EHg39x9e3uHZmiLyjmX5H0u13CP7Bqt7r4h+L4ZeJhUN0t98KcawffNweFR+j109hzXBdsHtpcNd69394S7J4G72delFolzNrMepELuz+7+UNAc6fc50zmX6n0u13CP5BqtZtbXzPq3bgOfARaROrcpwWFTgL8F248Al5tZLzMbC4wn9UFMOerUOQZ/0u8ws1OCkQRfTXtOWWgNucAlpN5riMA5B/X9AVjq7r9I2xXZ9znbOZfsfS71J8wFfDJ9PqlPo1cC15W6niKd05GkPj1fACxuPS9gCDAbeDv4PjjtOdcFv4NldNNRBBnOcyapP0+bSV2lXJnPOQK1wf8oK4HbCe647o5fWc75T8CbwMLgf/QRUTln4JOkuhIWAvODr/Oj/D63c84leZ81/YCISASVa7eMiIi0Q+EuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYmg/w+z0LxVtHhI+wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure('train')\n",
    "plt.plot(x,y)\n",
    "plt.savefig(\"train_distribution.jpg\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "250_percent: 0.6830204081632653\n",
      "500_percent: 0.9128979591836734\n",
      "1000_percent: 0.9941632653061224\n"
     ]
    }
   ],
   "source": [
    "n_1000 = 0\n",
    "n_500 = 0\n",
    "n_250 = 0\n",
    "n = 0\n",
    "for i,case in enumerate(x):\n",
    "    if case<=250:\n",
    "        n_250 = n_250 + y[i]\n",
    "    if case<=500:\n",
    "        n_500 = n_500 + y[i]\n",
    "    if case<=1000:\n",
    "        n_1000 = n_1000 + y[i]\n",
    "    n = n + y[i]\n",
    "print('250_percent:',n_250/n)\n",
    "print('500_percent:',n_500/n)\n",
    "print('1000_percent:',n_1000/n)# from these percents, we choose 500 as our MAX_SEQUENCE_LENGTH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_test = r'D:\\v-yanx\\masijia\\text_mood_classification\\text_mood_classification\\public_data\\testData.tsv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ids = []\n",
    "sentances = []\n",
    "with open(path_test, encoding='utf8') as f:\n",
    "    data = csv.reader(f,delimiter='\\t')\n",
    "    for line in data:\n",
    "        ids.append(line[0])\n",
    "        sentances.append(line[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'So im not a big fan of Boll\\'s work but then again not many are. I enjoyed his movie Postal (maybe im the only one). Boll apparently bought the rights to use Far Cry long ago even before the game itself was even finsished. <br /><br />People who have enjoyed killing mercs and infiltrating secret research labs located on a tropical island should be warned, that this is not Far Cry... This is something Mr Boll have schemed together along with his legion of schmucks.. Feeling loneley on the set Mr Boll invites three of his countrymen to play with. These players go by the names of Til Schweiger, Udo Kier and Ralf Moeller.<br /><br />Three names that actually have made them selfs pretty big in the movie biz. So the tale goes like this, Jack Carver played by Til Schweiger (yes Carver is German all hail the bratwurst eating dudes!!) However I find that Tils acting in this movie is pretty badass.. People have complained about how he\\'s not really staying true to the whole Carver agenda but we only saw carver in a first person perspective so we don\\'t really know what he looked like when he was kicking a**.. <br /><br />However, the storyline in this film is beyond demented. We see the evil mad scientist Dr. Krieger played by Udo Kier, making Genetically-Mutated-soldiers or GMS as they are called. Performing his top-secret research on an island that reminds me of \\\\SPOILER\\\\\" Vancouver for some reason. Thats right no palm trees here. Instead we got some nice rich lumberjack-woods. We haven\\'t even gone FAR before I started to CRY (mehehe) I cannot go on any more.. If you wanna stay true to Bolls shenanigans then go and see this movie you will not be disappointed it delivers the true Boll experience, meaning most of it will suck.<br /><br />There are some things worth mentioning that would imply that Boll did a good work on some areas of the film such as some nice boat and fighting scenes. Until the whole cromed/albino GMS squad enters the scene and everything just makes me laugh.. The movie Far Cry reeks of scheisse (that\\'s poop for you simpletons) from a fa,r if you wanna take a wiff go ahead.. BTW Carver gets a very annoying sidekick who makes you wanna shoot him the first three minutes he\\'s on screen.\"'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentances[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_dic1 = {}\n",
    "for i in range(1,len(sentances)):\n",
    "    text = ''.join([c if c.isalpha() else ' ' for c in sentances[i]])\n",
    "    words = [w for w in text.split(' ') if w]\n",
    "    key = '{:0>5}_{}'.format(i,ids[i])\n",
    "    words_dic1[key] = words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('test_words_dic.npy',words_dic1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_dic_test = np.load('test_words_dic.npy',allow_pickle=True).item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_lens = []\n",
    "for key in words_dic_test:\n",
    "    l = len(words_dic_test[key])\n",
    "    test_lens.append(l)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22000"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_lens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAf5klEQVR4nO3de3xU9Z3/8deHEML9JgERRC6iFm9IU9RqKdpaUfvTattd3K7VbrvUqvvobi+7WLutravrau3FtdpqxWp1tVq12q03ZLHewUC5IzcJEMAQ7pdArp/fHzkJk8kMSWbOZDJn3s/HI4/MfM+ZmU/OI7zz5Zzv+X7N3RERkWjplu0CREQkfAp3EZEIUriLiESQwl1EJIIU7iIiEaRwFxGJoDbD3cxmmdk2M1sW0/Z7M1sUfJWZ2aKgfbSZHYzZ9qtMFi8iIol1b8c+vwXuAR5panD3v216bGZ3AXti9l/n7hPDKlBERDquzXB399fNbHSibWZmwN8A54dbloiIpKM9Pfcj+QRQ4e5rYtrGmNlfgb3A9939jbbeZMiQIT569Og0SxERyS8LFizY7u7FibalG+5XAo/HPN8KjHL3HWb2UeCPZnayu++Nf6GZzQBmAIwaNYrS0tI0SxERyS9mtiHZtpRHy5hZd+AK4PdNbe5e7e47gscLgHXACYle7+73u3uJu5cUFyf8wyMiIilKZyjkp4H33b28qcHMis2sIHg8FhgPfJBeiSIi0lHtGQr5OPAOcKKZlZvZV4NN02l5SgZgCrDEzBYDfwCudfedYRYsIiJta89omSuTtF+ToO1p4On0yxIRkXToDlURkQhSuIuIRJDCXUQkghTuwMKNu1ixpdVQfBGRnJXuTUyRcMW9bwNQdvslWa5ERCQc6rmLiESQwl1EJIIU7iIiEaRwFxGJIIW7iEgEKdxFRCJI4S4iEkEK9xivr67MdgkiIqHI+3B/c8325scLN+7KYiUiIuHJ+3B/6K31zY8LC/L+cIhIROR9mnXrZs2Py3dVZbESEZHw5H24F9jhcH98/qYsViIiEh6Fe0zPXUQkKvI+3Lsp3EUkgvI+3AuU7SISQXkf7uq5i0gUtRnuZjbLzLaZ2bKYtpvNbLOZLQq+Lo7ZdqOZrTWzVWZ2YaYKD4uhcBeR6GlPz/23wLQE7T9z94nB1wsAZjYBmA6cHLzmXjMrCKvYTHA82yWIiISuzXB399eBne18v8uAJ9y92t3XA2uByWnUl3GubBeRCErnnPsNZrYkOG0zKGgbAcQOFi8P2loxsxlmVmpmpZWV2ZvTxZXuIhJBqYb7fcA4YCKwFbgraE90Ajtherr7/e5e4u4lxcXFKZaRPkW7iERRSuHu7hXuXu/uDcADHD71Ug4cG7PrSGBLeiVm1nOLunR5IiIpSSnczWx4zNPLgaaRNM8D082syMzGAOOB+emVKCIiHdW9rR3M7HFgKjDEzMqBHwJTzWwijWc1yoCvA7j7cjN7ElgB1AHXu3t9ZkoXEZFk2gx3d78yQfODR9j/VuDWdIoSEZH05P0dqiIiUaRwj/G3Jce2vZOISA7I+3Af0rco2yWIiIQu78O9b9Hh2RE0FYGIREXeh3tsnD9ZWs7eQ7VZq0VEJCwK97jO+o1PL81OISIiIVK4x52KqdxXnaVKRETCo3CP67nXayIxEYmAvA/38l0HWzxvULiLSATkfbjHa1C2i0gEKNzjaH53EYkChXucenXdRSQCFO5xFO4iEgUK9zi19Q3ZLkFEJG0K9zg1CncRiQCFe5zqWoW7iOQ+hXscnZYRkShQuMepqVO4i0juU7jH0Tl3EYkChXuc2noNhRSR3KdwFxGJoDbD3cxmmdk2M1sW03anmb1vZkvM7FkzGxi0jzazg2a2KPj6VSaLT9eO/Ymn99V5dxHJde3puf8WmBbXNhs4xd1PA1YDN8ZsW+fuE4Ova8MpMzPmrNyWsH3rnoMJ20VEckWb4e7urwM749pecfe64Om7wMgM1JZxuw/WJGzvXqCzVSKS28JIsX8AXox5PsbM/mpmfzGzTyR7kZnNMLNSMyutrKwMoYyOu+2F9xO2V+w91MmViIiEK61wN7ObgDrgsaBpKzDK3c8AvgX8j5n1T/Rad7/f3UvcvaS4uDidMkJ3xb1vZ7sEEZG0pBzuZnY18FngSx5Mgu7u1e6+I3i8AFgHnBBGoZk2ZkifbJcgIhKalMLdzKYB/wZc6u5VMe3FZlYQPB4LjAc+CKPQTBvcp0e2SxARCU17hkI+DrwDnGhm5Wb2VeAeoB8wO27I4xRgiZktBv4AXOvuOxO+cRcyrrgP9/39pGyXISISmu5t7eDuVyZofjDJvk8DT6dbVGf71gUnMrRfT84eexTvfLAj2+WIiKRNY/5iXH7GiGyXICISCoV7DLNsVyAiEg6FewxTuotIRCjcAadxJkhFu4hEhcI9RjcdDRGJCMVZDFPfXUQiQuEeI/aUe3DTrYhITlK4x4i9oKpsF5FcpnCPEXtSpkHpLiI5TOEeo1tMz/0Hzy/PYiUiIulRuMeIPef+P/M2Zq8QEZE0Kdw5fH69W9xgmfJdVa13FhHJAQr3Flqm+9ceLs1SHSIi6cnbcD9UW9+qLX72gfJdWihbRHJT3oZ7Ih/uabl26v7quiR7ioh0bXkb7olGOm7fX92qbf76Lr/WiIhIK3kb7rHj2AsLkk878MryDzujHBGRUCncgQsmHA3AtZ8c12o/3cokIrkob8O9KbRPGdGfgmAMZJ+iNlcdFBHJCfkb7g2N3684Y+SR91PXXURyUN6Ge9NpmbYWX3KdmBGRHNRmuJvZLDPbZmbLYtoGm9lsM1sTfB8Us+1GM1trZqvM7MJMFZ6upsju1ka6P/RWWcZrEREJW3t67r8FpsW1zQTmuPt4YE7wHDObAEwHTg5ec6+ZFYRWbYiaeu7xUw6IiERBm+Hu7q8D8YO9LwMeDh4/DHwupv0Jd6929/XAWmBySLWGqnm0jBbFFpEISvWc+zB33woQfB8atI8ANsXsVx60tWJmM8ys1MxKKysrUywjDUkmCxMRiYKwL6gmisqEVyTd/X53L3H3kuLi4pDLaFtDc7i3LHnGlLGdXouISNhSDfcKMxsOEHzfFrSXA8fG7DcS2JJ6eZnTPFomrl1rp4pIFKQa7s8DVwePrwaei2mfbmZFZjYGGA/MT6/EzDh8QbVlvCvbRSQK2rwl08weB6YCQ8ysHPghcDvwpJl9FdgIfBHA3Zeb2ZPACqAOuN7dW8+t2wUku56qbBeRKGgz3N39yiSbPpVk/1uBW9MpqjM8taAcaF/Pva6+ge4FeXu/l4jkoLxNrLvnrAES9dxbp/vi8t2dUZKISGjyNtybtKfnXl3X0EnViIiEI+/DvT33MM18emnmCxERCZHCvVXPvXXXfePOqs4qR0QkFAr3uOcaLSMiUZD34V7QzvkHfv2Xddz72toMVyMiEo68X3qovTcx/eeL7wNw3dTjM12SiEja1HOP67lrcQ4RiQKFe9wR0PQDIhIFeR/urU7LZKkOEZEw5X2419a3jHP13EUkCvI+3Ktq6uJaGtP9qrOO6/xiRERCkvfh3vompsbvJx7dLwvViIiEIy/DPfYu1PqGhrhtjd/bO/5dRKQrystwb4g5r14fNydY01BIRbuI5LI8DffD6d7QkPiCarIJxbQMn4jkgrwP9+J+RS22NW2xJH33u+doCgIR6fryMtwfeXtD8+PzThraYltbPfefvbo6U2WJiIQmL8P91hdWJt3WfM69PRO9i4h0UXkZ7kfyrxeexIUnD+PiU4/OdikiIilLeVZIMzsR+H1M01jgB8BA4B+ByqD9e+7+QsoVdrKjB/Tk11eVZLsMEZG0pBzu7r4KmAhgZgXAZuBZ4CvAz9z9J6FUKCIiHRbWaZlPAevcfUObe4qISMaFFe7Tgcdjnt9gZkvMbJaZDQrpM0REpJ3SDncz6wFcCjwVNN0HjKPxlM1W4K4kr5thZqVmVlpZWZloFxERSVEYPfeLgIXuXgHg7hXuXu/uDcADwOREL3L3+929xN1LiouLQyhDRESahBHuVxJzSsbMhsdsuxxYFsJniIhIB6S1QLaZ9QYuAL4e03yHmU2k8U7+srhtIiLSCdIKd3evAo6Ka7sqrYpERCRtukNVRCSCFO4iIhGkcBcRiSCF+xF898ITs12CiEhKFO5HcP15xyds37bvUCdXIiLSMQr3NsyYMrZV2+Rb51DfoOX2RKTrUri3IdmSHdv3V3dqHSIiHaFwb0uSdK+ubejcOkREOiCvw/3LZx/X5j7JFsqOXWRbRKSryetwLxk9uM19CgsU7iKSe/I63NuzBHavHgUJ2xXuItKV5XW4t0fvwmTh3smFiIh0QF6Hu7Wj6967R+K51R56qyzcYkREQpTf4d6OEzO9ixL33B+fv5FDtfVhlyQiEorIhntVTR3f/+NS9lfXJd2nfT33xOEOGusuIl1XZMP9obfKePTdjZz/k9eS3k3anguqHx83hIJuifesqdNYdxHpmiIb7k227avm9+9tSritPT33noUFvDPzfAB+8NkJLbYtLt/NgSP8z0BEJFsiG+519Yd763sP1SbZqz19dxjavydlt1/CueOHtGj/l98v5rrHFqZaoohIxkQy3PdU1fKzV1c3P29fhLetW4Ku/rz1O0J6dxGR8EQy3Cv3t5yS9+45a5j289fTft9Ep94LCyJ5CEUkx0U0mVqm8IGaet7/cB8AZdsPNLfvS3q6JrFEPfd9h+pw3a0qIl1MWuFuZmVmttTMFplZadA22Mxmm9ma4PugcErtSF3Jt33nqcXNjy8+dXiH3jdRuANU7NWQSBHpWsLouZ/n7hPdvSR4PhOY4+7jgTnB806VLNuXlO+mR/fDP3KfosR3nybTLcnRWly+m6oajZoRka4jE6dlLgMeDh4/DHwuA59xRJakh33pPW8l7X2n875f/90CvvnEopTfV0QkbOmGuwOvmNkCM5sRtA1z960AwfehaX5GqJLdkNQeTefWB/QqbLWttGxnyu8rIhK2dMP9HHefBFwEXG9mU9r7QjObYWalZlZaWVmZZhkt3fTs0qTbYk/LdFTTddNeCWaK3FVVy3OLNqf83iIiYUor3N19S/B9G/AsMBmoMLPhAMH3bUlee7+7l7h7SXFxcTpltPL2uuRjz8MI92S9/3v+b23K7y0iEqaUk87M+phZv6bHwGeAZcDzwNXBblcDz6VbZJiWlO9O+bVNC3R0T7I6Uzrn80VEwpROz30Y8KaZLQbmA39295eA24ELzGwNcEHwvMvYtPNgyq8dPrAn44r78OPLTkm4vVsa5/NFRMLUsbGAMdz9A+D0BO07gE+lU1RXVdS9gDnfngrAuOI+rKs80GK7sl1EuoqI3qGaeYnuSU1nJI6ISJgU7iFKNg5eRKSzKdxTlaDrnuQ6q4hIp1O4h6ibGZt2VvHfc9ZoMjERySqFe4ru+MJprdoa3Pn2U4u5a/ZqVlXsy0JVIiKNFO4pKhk9mAsmDGvRtmbb/uZJy7ZppkgRySKFexrOHDO4xfN9h+qYt75xjpmqmvpslCQiAijc0/J3Z45Kuu1g7eEpgC+9501Ou/nlzihJRARI4yYmgd49kh++gzUNfLjnENc8NL95FSgRkc4S+Z57toaeV9XUsXDjrhbBrhE0ItJZIh/uxxf3zcrn/sefV/LMwpZTAMdPVyAikimRD/cThvXL2me/urKixXPdwCoinSXS4X7b5adyyWkdWwQ7kzQlsIh0lkhfUL3w5GEc1bcoo5/Rt6g7+6vbtzi2ol1EOkuke+5NszS++W/nZewzXvvuVF79VvtWF5z6k9d4ekF5xmoREWkSuXBfvmVP8+OmxTNGDuqdsc8b0reI44e2/7z+D55blrFaRESaRO60zN89MK/5cfcE86sX9yvi+qnjOHpAr84sq9mBmnoO1tTTq0frRbZFRMISuZ57bJ4nuoDZzeCac8Yw7ZSjQ/3ch675WLv3verBeRyqbd/0BIs37W73viIiTSIX7gXdDv9IiXrulqHLmlNOKG73vqUbdvGNRxewp6r2iDc2bdt3iMt++Rbf/cOSMEoUkTwSuXC3NnrumRqN2NEV9uauquT0H7/CrLfKku6z/1DjKJyl5bvTqExE8lHkwr2pJzx5zODmC6qxMjXWPNUl9l5Z/mHIlYiIpBHuZnasmc01s5VmttzMvhm032xmm81sUfB1cXjltq3pLMe44j4Jtxd2sbXw4k/K3PTsUt5YU9lim9ZmFZGOSqfnXgd8290/ApwFXG9mE4JtP3P3icHXC2lX2QFtTc310Fcmd0od7RZX8GPzNnLVg/NbtCnaRaSjUg53d9/q7guDx/uAlcCIsApLVdNpmfjrlP827STOP2koY4Yk7tGH7ZPtvMDqbf45EhHpuFDOuZvZaOAMoGmQ+Q1mtsTMZpnZoCSvmWFmpWZWWllZGUYZQPKe+zemjmNWB4YrpuuLJSPbtd/WPYea/yA1NLSsXjMEi0iq0g53M+sLPA38s7vvBe4DxgETga3AXYle5+73u3uJu5cUF7d/GGFbmgIxG8FYctzhv2OfPe2Ydr2mfNdBHgpGzNQnK1rnZUSkg9IKdzMrpDHYH3P3ZwDcvcLd6929AXgA6NST3M2nZbJwuuORr6b2o/74f1cweuafGX/Ti3Fb1HUXkdSkM1rGgAeBle7+05j22Dl2Lwc6dTKVwoLGH6moe+ff3h+/7F5R99T/do6e+WdWV+wH1HEXkY5Lp+d+DnAVcH7csMc7zGypmS0BzgP+JYxC2+uzwfzt3512Ymd+bEJ/+qdz+fRHhnLXF09P6fXfe3Yp0DgU8kB1HXe89D41dQ1hligiEZXyxGHu/iaJO5WdOvQxXrduRr+i7vTvWZjNMoDGVaB+c/XHqG9wvv3U4g6/vrr2cJBPv/9dlm7ew4hBvfjSmceFWaaIRFDkZoV073rL2XV0aoImB4MJw9Zu29/cVhv03NdvP9BpwzpFJPdEcvqBRNMOdJaPHtd65KeZUXb7JZTdfkna729mvLlmO+f95DWeWaiFP0Qkscj13Bs8uxcgH/vamRxo57J7qfjh88v58tmNp2UWbdrNFZPaN55eRPJL9HrueFYXou5ZWHDEdVubevYzpoxN+TOWb9kLQG29hkqKSGKRCnd359F3N7LjQE22S0nqE+OHANAzjWGSTUMs6+o7PnJmXeV+fjp79RHnkReR3BepcK+q6forFl3z8dF8+iPD+PuzUx/xsnn3QQDe+WAH9762loq9h1rts3FHVfPskrGu+s087p6zhu37u+4fQBFJX6TCva6h6/dGB/buwW+uLmFov54pv8eGHVVA49QFd7y0iivufZvlW/ZwoLqODyr3U1PXwJQ757aaXRKgWuPkRfJCpC6o1qZwmqKr6VHQjZoO/hybdx/kkrvfbH5+5eRjk+57eI741tsaGpwGd7oXROpvvkheitS/4qZwn3nRSVmuJHUrb5nGWzPPT+s95q3f2eY+iS45X/nAuxzfan4bEclFkQr35xZtAWBgr+zfndoR//G5U5ofF3QzRgzsldb7fVB5oMXz5xZt5qzb5rBpZ1XMxGqtteePgojkhkiF++0vvg/AvkOZG2eeCVdOHpXR93/03Q18uPcQn7hjLruqaoHDUyK/sHQrj83bkNHPF5HOF6lwb9LV1kltS0GCO2r7FYVzOWT5lj28V7arVXtTD/66xxZy07PLWgyr/Mvq8BZPEZHsiGS45+IFwa9PGdti6oJbYk7VpCP2Qmusm/64jAUbDof+z19d0/z46lmtR9mISG6J1GiZJtm8Q7UjRg7qxYd7Gseo33jxR1ps+9wZI3hs3oaEve4wzF5RwewVFc3P75m7tsX2vYdq6VfUHTPD3dlXXUeBGTV1DQzsXYglOcYHa+rpXmDN8+qLSHZE8l9gFucN65DXvjOVlbdMS7r9iRlnJ2w/a+zg0E7bJHPaza8wK1j+79F3N3Daza9w8g9f5oxbZjcvC5jIR37wEv/w2/cyWpuItC2S4Z4jHXe6F3Q7Yg83/lz8L6ZPBOD4oX350z+d29x+wrC+GanvlmD5v39/bnmL9qZlAafeOTfh695Ys71V28Nvl3Hi91/UtAcinSQy4R4bGrlyWqY9vnXBCc2PLz39GG6/4lS+f8kERsfM5T58QHpDJ1NVtqOKl5d/2K59b/7TcqrrGjhUm/0bzXYdqOEbjy5gd5WmYJDoiky4x84rc+nEY7JYSbjOP2koABOG98fMmD55FD0LW64Pe/vnT231uu9e2DnLDH79dwuYvaKCTTuruO6xBc3tpWU7eXVFBSu27OWz//1G89DLfYdqaWhwfvWXdew71Dgs81BtPb+cuzbjSwjW1jfwy7lrOe+u13hx2Yc88o6GgEp0ReaC6s5gJsg7vnBaVhbHzpRxxX05ZkBPvhd3wTVWfM99yc2foX/PQu58eVWmywPgHx8pZfRRvSkL5rwB+MKv3km474NvrmfO+9tYu20/s1dUMGpwb/66cRdlO6qYt34nD15dwjvrdtC/VyEjB/WiV2EBSzfvYdKoQcxfv5Nzxw+hocF5fU0lZsbk0YN5r2wnU04obvE5736wg1NHDKBPcG3ivtfWcai2nl/MOTwqaNPOKpZt3sMpIwZk4KiIZFckwt3dm8/zDu7dI8vVhKtXjwLevvFTCbedNnIAqyv2tWrPxvqxscF+JL9+/YPmxws27GoxHPP11ZWceduc5j/UY4b0oaaugc27D3LCsL6srtjPM9d9nMWbdvOjP61o8b7PXPdxJo0axMYdVRQVdmP6/e9y4cnDuOMLp/Pe+p3810vvt6rlqQXlPLWgnDf+9TyK+xW1+h+RSC7L+dMy+6vreHPtdr737FIABvWJVrgfyfM3nMv7t1zUoi12Kb9rPj66xbaPjU48jv6D2y7OTIEp2BkzF//67QeapzdeXdG4juz2fdXNs2LGemfdDjbtrGLKnXO5+fnGC8CLNu3m9B+9wtceKT3iZ37ijrn85wsrw/oRRLqEjPXczWwa8AugAPiNu98e9mfMX7+Tv/l1y//+D+2XfBWkKFvw/U+3Gns+86KT+PykkRw9oCcVew8xtrgP//7H5Ty9sJxjBhyecjiba8521IzfLUjYfufLq5pPQ724rPEib8Xe6na/75z3t/Gjy9KvT6SrsEwMTTOzAmA1cAFQDrwHXOnuKxLtX1JS4qWlR+5dJbLnYC2n/+iVFm1hLEIdZTV1Dby0/EP+32nDWVd5gL2Hapk0ahDPLCznW08uBmDud6by2qptHDOwFwN7FfK/S7byu3fbvvg4adRAFm7cnekfISMKC4zRR/Vpe0eRkE09sZibLpmQ0mvNbIG7lyTalqme+2Rgrbt/EBTwBHAZkDDcUzWgVyHXfnIcj727gcF9e3Db5a1HjUhLPbp349LTG0cTHT/08Pj4KyaN5LSRA5izchtjhvRhzJAxzdsmjxnM2+u285VzxtCrsICh/YuYs3IbVTV1nDpyIIN792DRpl18+zMnUrbjANN+/gZA83nyJhOPHUhxv6IWd8Z+ftJIKvdXc3T/Ip4sLW9uv/aT45h2ytHMX7+De/5vLXsTTAb3yROKOfHofgzo1XjxeNKogSwp30PPwgL2xyxSPqx/EeccP4RnFm4GYMTAXnzprFHc8dIqxg/tS4/u3TjuqN7pHlqRlAzrn/rCPUeSqZ77F4Bp7v614PlVwJnufkPMPjOAGQCjRo366IYNGpYmItIRR+q5Z+qCaqKTuC3+irj7/e5e4u4lxcXFCXYXEZFUZSrcy4HYtd5GAlsy9FkiIhInU+H+HjDezMaYWQ9gOvB8hj5LRETiZOSCqrvXmdkNwMs0DoWc5e7L23iZiIiEJGPj3N39BeCFTL2/iIgkl/N3qIqISGsKdxGRCFK4i4hEUEZuYupwEWaVQKp3MQ0BWi/9k990TBLTcUlMx6W1XDkmx7l7whuFukS4p8PMSpPdoZWvdEwS03FJTMeltSgcE52WERGJIIW7iEgERSHc7892AV2QjkliOi6J6bi0lvPHJOfPuYuISGtR6LmLiEicnA13M5tmZqvMbK2Zzcx2PZ3NzMrMbKmZLTKz0qBtsJnNNrM1wfdBMfvfGByrVWZ2YfYqD4+ZzTKzbWa2LKatw8fAzD4aHMu1Zna3xa9XmGOSHJebzWxz8PuyyMwujtkW+eNiZsea2VwzW2lmy83sm0F7dH9f3D3nvmicjGwdMBboASwGJmS7rk4+BmXAkLi2O4CZweOZwH8FjycEx6gIGBMcu4Js/wwhHIMpwCRgWTrHAJgPnE3jOgQvAhdl+2fLwHG5GfhOgn3z4rgAw4FJweN+NC4DOiHKvy+52nNvXsbP3WuApmX88t1lwMPB44eBz8W0P+Hu1e6+HlhL4zHMae7+OrAzrrlDx8DMhgP93f0db/yX+0jMa3JSkuOSTF4cF3ff6u4Lg8f7gJXACCL8+5Kr4T4C2BTzvDxoyycOvGJmC4IlCwGGuftWaPxlBoYG7fl0vDp6DEYEj+Pbo+gGM1sSnLZpOv2Qd8fFzEYDZwDziPDvS66Ge5vL+OWBc9x9EnARcL2ZTTnCvjpeyY9Bvhyb+4BxwERgK3BX0J5Xx8XM+gJPA//s7nuPtGuCtpw6Lrka7nm/jJ+7bwm+bwOepfE0S0Xw30aC79uC3fPpeHX0GJQHj+PbI8XdK9y93t0bgAc4fFoub46LmRXSGOyPufszQXNkf19yNdzzehk/M+tjZv2aHgOfAZbReAyuDna7GnguePw8MN3MisxsDDCexotCUdShYxD8V3yfmZ0VjHr4csxrIqMpwAKX0/j7AnlyXIKf4UFgpbv/NGZTdH9fsn1FN9Uv4GIar3ivA27Kdj2d/LOPpfFK/mJgedPPDxwFzAHWBN8Hx7zmpuBYraKLXt1P4Tg8TuMphloae1RfTeUYACU0ht064B6Cm/ty9SvJcfkdsBRYQmNwDc+n4wKcS+PpkyXAouDr4ij/vugOVRGRCMrV0zIiInIECncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIuj/A5o3KznJEt//AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "t_uni = set(test_lens)\n",
    "t_x = []\n",
    "t_y = []\n",
    "for i in t_uni:\n",
    "    t_x.append(i)\n",
    "t_x.sort()\n",
    "for i in t_x:\n",
    "    t_y.append(test_lens.count(i))\n",
    "plt.figure('test')\n",
    "plt.plot(t_x,t_y)\n",
    "plt.savefig(\"test_distribution.jpg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "250_percent: 0.6974545454545454\n",
      "500_percent: 0.9182272727272728\n",
      "1000_percent: 0.9953636363636363\n",
      "t_n: 22000\n"
     ]
    }
   ],
   "source": [
    "t_n_1000 = 0\n",
    "t_n_500 = 0\n",
    "t_n_250 = 0\n",
    "t_n = 0\n",
    "for i,case in enumerate(t_x):\n",
    "    if case<=250:\n",
    "        t_n_250 = t_n_250 + t_y[i]\n",
    "    if case<=500:\n",
    "        t_n_500 = t_n_500 + t_y[i]\n",
    "    if case<=1000:\n",
    "        t_n_1000 = t_n_1000 + t_y[i]\n",
    "    t_n = t_n + t_y[i]\n",
    "print('250_percent:',t_n_250/t_n)\n",
    "print('500_percent:',t_n_500/t_n)\n",
    "print('1000_percent:',t_n_1000/t_n)\n",
    "print('t_n:',t_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}