Yelp-Mining_with_DR_NMF.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from util import *\n",
    "import time\n",
    "import datetime\n",
    "\n",
    "import cPickle as pickle\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pylab\n",
    "import re\n",
    "import scipy as sp\n",
    "import seaborn\n",
    "import sklearn.feature_extraction.text as text\n",
    "\n",
    "from gensim import corpora, models\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn import decomposition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lines=[]\n",
    "with open(\"data/reviews_full.dat\", \"r\") as fh:\n",
    "    lines = fh.readlines()  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "userid = []\n",
    "rating = []\n",
    "docs = []\n",
    "business = []\n",
    "i = 0\n",
    "j = 0\n",
    "error_line_num = []\n",
    "error_lines = []\n",
    "for line in lines:\n",
    "    try:\n",
    "        i = i + 1\n",
    "        l = line.split('\\t', 4)\n",
    "        userid.append(l[0])\n",
    "        business.append(l[1])\n",
    "        rating.append(l[2])\n",
    "        docs.append(l[3])\n",
    "        #d = clean(l[3])\n",
    "        #kmers = getKmers(d)\n",
    "        #d.extend(kmers)\n",
    "        #docs.append(d)\n",
    "    except Exception as e:\n",
    "        j = j + 1\n",
    "        error_line_num.append(i)\n",
    "        error_lines.append(line)\n",
    "\n",
    "print 'Training Data: Number of lines processed: ' + str(i)\n",
    "print 'Training Data: Length of userid array: ' + str(len(userid))\n",
    "print 'Training Data: Length of rating array: ' + str(len(rating))\n",
    "print 'Training Data: Length of docs array: ' + str(len(docs))\n",
    "print 'Training Data: Length of business array: ' + str(len(business))\n",
    "print 'Training Data: Number of exceptions encountered: ' + str(j)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "from time import time\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.decomposition import NMF, LatentDirichletAllocation\n",
    "n_samples = 50000\n",
    "n_features = 15000\n",
    "n_components = 30\n",
    "n_top_words = 50\n",
    "\n",
    "data_samples = docs[:n_samples]\n",
    "\n",
    "def print_top_words(model, feature_names, n_top_words):\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "        message = \"Topic #%d: \" % topic_idx\n",
    "        message += \" \".join([feature_names[i]\n",
    "                             for i in topic.argsort()[:-n_top_words - 1:-1]])\n",
    "        print(message)\n",
    "    print()\n",
    "\n",
    "\n",
    "# Use tf-idf features for NMF.\n",
    "print(\"Extracting tf-idf features for NMF...\")\n",
    "tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=2,\n",
    "                                   max_features=n_features,\n",
    "                                   stop_words='english')\n",
    "t0 = time()\n",
    "tfidf = tfidf_vectorizer.fit_transform(data_samples)\n",
    "print(\"done in %0.3fs.\" % (time() - t0))\n",
    "\n",
    "\n",
    "# Fit the NMF model\n",
    "print(\"Fitting the NMF model (generalized Kullback-Leibler divergence) with \"\n",
    "      \"tf-idf features, n_samples=%d and n_features=%d...\"\n",
    "      % (n_samples, n_features))\n",
    "t0 = time()\n",
    "nmf = NMF(n_components=n_components, random_state=1,\n",
    "          beta_loss='kullback-leibler', solver='mu', max_iter=1000, alpha=.1,\n",
    "          l1_ratio=.5).fit(tfidf)\n",
    "\n",
    "tfidf_nmf = nmf.transform(tfidf)\n",
    "\n",
    "print(\"done in %0.3fs.\" % (time() - t0))\n",
    "\n",
    "print(\"\\nTopics in NMF model (generalized Kullback-Leibler divergence):\")\n",
    "#tfidf_feature_names = tfidf_vectorizer.get_feature_names()\n",
    "vocab = np.array(tfidf_vectorizer.get_feature_names())\n",
    "#print_top_words(nmf, tfidf_feature_names, n_top_words)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Data: Number of lines processed: 317586\n",
      "Training Data: Length of userid array: 317586\n",
      "Training Data: Length of rating array: 317586\n",
      "Training Data: Length of docs array: 317586\n",
      "Training Data: Length of business array: 317586\n",
      "Training Data: Number of exceptions encountered: 0\n"
     ]
    }
   ],
   "source": [
    "from util import *\n",
    "import time\n",
    "import datetime\n",
    "\n",
    "import cPickle as pickle\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pylab\n",
    "import re\n",
    "import scipy as sp\n",
    "import seaborn\n",
    "import sklearn.feature_extraction.text as text\n",
    "\n",
    "from gensim import corpora, models\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn import decomposition\n",
    "\n",
    "lines=[]\n",
    "with open(\"data/reviews_full.dat\", \"r\") as fh:\n",
    "    lines = fh.readlines()  \n",
    "    \n",
    "userid = []\n",
    "rating = []\n",
    "docs = []\n",
    "business = []\n",
    "i = 0\n",
    "j = 0\n",
    "error_line_num = []\n",
    "error_lines = []\n",
    "for line in lines:\n",
    "    try:\n",
    "        i = i + 1\n",
    "        l = line.split('\\t', 4)\n",
    "        userid.append(l[0])\n",
    "        business.append(l[1])\n",
    "        rating.append(l[2])\n",
    "        docs.append(l[3])\n",
    "        #d = clean(l[3])\n",
    "        #kmers = getKmers(d)\n",
    "        #d.extend(kmers)\n",
    "        #docs.append(d)\n",
    "    except Exception as e:\n",
    "        j = j + 1\n",
    "        error_line_num.append(i)\n",
    "        error_lines.append(line)\n",
    "\n",
    "print 'Training Data: Number of lines processed: ' + str(i)\n",
    "print 'Training Data: Length of userid array: ' + str(len(userid))\n",
    "print 'Training Data: Length of rating array: ' + str(len(rating))\n",
    "print 'Training Data: Length of docs array: ' + str(len(docs))\n",
    "print 'Training Data: Length of business array: ' + str(len(business))\n",
    "print 'Training Data: Number of exceptions encountered: ' + str(j)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "\n",
    "total_data =50000\n",
    "train_data =40000\n",
    "\n",
    "def getSentiment(x):\n",
    "    if x < 3.0:\n",
    "        return 0\n",
    "    else:\n",
    "        return 1\n",
    "    \n",
    "\n",
    "def nmf(rating):\n",
    "    nmf = load_pickle('nmf.pickle')\n",
    "    print nmf.shape\n",
    "    trainTopics= nmf[:total_data]\n",
    "    rating=rating[:total_data]\n",
    "    print trainTopics.shape\n",
    "    print len(rating)\n",
    "    trainTopics = trainTopics / np.sum(trainTopics, axis=1, keepdims=True)\n",
    "    d, f = trainTopics.shape\n",
    "    cols = [\"Topic\"+str(i) for i in xrange(1, f+1)]\n",
    "    nmfDF = pd.DataFrame(trainTopics, columns=cols)\n",
    "    nmfDF['rating'] = map(float,rating)\n",
    "    cols = [u'Topic1', u'Topic2', u'Topic3', u'Topic4', u'Topic5', u'Topic6', u'Topic7', u'Topic8', u'Topic9', u'Topic10', u'Topic11', u'Topic12', u'Topic13', u'Topic14', u'Topic15', u'Topic16', u'Topic17', u'Topic18', u'Topic19', u'Topic20']\n",
    "    Xtrain = nmfDF[:train_data][cols]\n",
    "    Ytrain = nmfDF[:train_data]['rating']\n",
    "    Xtest = nmfDF[train_data:][cols]\n",
    "    Ytest = nmfDF[train_data:]['rating']\n",
    "    \n",
    "    return nmfDF, Xtrain, Ytrain, Xtest, Ytest\n",
    "    \n",
    "    \n",
    "def nmf_sentiment(rating):\n",
    "    nmfDF,a,b,c,d = nmf(rating)\n",
    "    nmfDF['Sentiment'] = nmfDF['rating'].map(getSentiment)\n",
    "    nmfDF = nmfDF.dropna(how='any')\n",
    "    cols = [u'Topic1', u'Topic2', u'Topic3', u'Topic4', u'Topic5', u'Topic6', u'Topic7', u'Topic8', u'Topic9', u'Topic10', u'Topic11', u'Topic12', u'Topic13', u'Topic14', u'Topic15', u'Topic16', u'Topic17', u'Topic18', u'Topic19', u'Topic20', u'Sentiment']\n",
    "    Xtrain = nmfDF[:train_data][cols]\n",
    "    Ytrain = nmfDF[:train_data]['rating']\n",
    "    Xtest = nmfDF[train_data:][cols]\n",
    "    Ytest = nmfDF[train_data:]['rating']\n",
    "    \n",
    "    return nmfDF,Xtrain,Ytrain,Xtest,Ytest\n",
    "\n",
    "    \n",
    "    \n",
    "def lda_sentiment(rating):\n",
    "    lda = load_pickle('lda.pickle')\n",
    "    trainTopics= lda[:total_data]\n",
    "    rating=rating[:total_data]\n",
    "    trainTopics = trainTopics / np.sum(trainTopics, axis=1, keepdims=True)\n",
    "    d, f = trainTopics.shape\n",
    "    cols = [\"Topic\"+str(i) for i in xrange(1, f+1)]\n",
    "    ldaDF = pd.DataFrame(trainTopics, columns=cols)\n",
    "    ldaDF['rating'] = map(float,rating)\n",
    "    ldaDF['Sentiment'] = ldaDF['rating'].map(getSentiment)\n",
    "    ldaDF = ldaDF.dropna(how='any')\n",
    "    cols = [u'Topic1', u'Topic2', u'Topic3', u'Topic4', u'Topic5', u'Topic6', u'Topic7', u'Topic8', u'Topic9', u'Topic10', u'Topic11', u'Topic12', u'Topic13', u'Topic14', u'Topic15', u'Topic16', u'Topic17', u'Topic18', u'Topic19', u'Topic20', u'Sentiment']\n",
    "    Xtrain = ldaDF[:train_data][cols]\n",
    "    Ytrain = ldaDF[:train_data]['rating']\n",
    "    Xtest = ldaDF[train_data:][cols]\n",
    "    Ytest = ldaDF[train_data:]['rating']\n",
    "    \n",
    "    return ldaDF,Xtrain,Ytrain,Xtest,Ytest\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "317586\n",
      "(50000, 20)\n",
      "(50000, 20)\n",
      "50000\n"
     ]
    }
   ],
   "source": [
    "rating = map(float,rating)\n",
    "print len(rating)\n",
    "df,train_X,train_Y,test_X,test_Y = nmf(rating)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 20)\n",
      "(50000, 20)\n",
      "50000\n",
      "(50000, 20)\n",
      "(50000, 20)\n",
      "50000\n",
      "Evalutaion of Model:  NMF\n",
      "********************************************************************\n",
      "Training validations evaluations for : Logistic Regression\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "\n",
    "\n",
    "from sklearn.metrics import median_absolute_error\n",
    "from sklearn.metrics import mean_squared_log_error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import explained_variance_score\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "\n",
    "\n",
    "ModelEvaluationResults = {}\n",
    "\n",
    "rating = map(float,rating)\n",
    "\n",
    "models= [nmf(rating),nmf_sentiment(rating),lda_sentiment(rating)]\n",
    "model_names = [\"NMF\",\"NMF-Sentiment\",\"LDA-Sentiment\"]\n",
    "\n",
    "\n",
    "results = {}\n",
    "\n",
    "for (m,models_) in enumerate(models):\n",
    "    df,train_X,train_Y,test_X,test_Y = models_\n",
    "    print \"Evalutaion of Model:  \"+str(model_names[m])\n",
    "    \n",
    "    clfs = [ LogisticRegression(),\n",
    "    KNeighborsRegressor(n_neighbors=3),\n",
    "    SVR(C=1.0, cache_size=200, coef0=0.0, degree=3, epsilon=0.2, gamma='auto',kernel='rbf', max_iter=-1, shrinking=True, tol=0.001, verbose=False)\n",
    "    ]\n",
    "    #KernelRidge(alpha=1.0, coef0=1, degree=3, gamma=None, kernel='linear',kernel_params=None)]\n",
    "\n",
    "    #clf_names = ['Logistic Regression','KNeighborsRegressor','SVR','KernelRidge']\n",
    "    clf_names = ['Logistic Regression','KNeighborsRegressor','SVR']\n",
    "\n",
    "\n",
    "    \n",
    "    ClassifierEvaluationResults = {}\n",
    "    \n",
    "    for (i, clf_) in enumerate(clfs):\n",
    "        #clf = clf_.fit(Xtrain, Ytrain)\n",
    "        #preds = clf_.predict(Xtest)\n",
    "        print \"********************************************************************\"\n",
    "        print \"Training validations evaluations for : \"+str(clf_names[i])\n",
    "        scores = cross_val_score(clf_, train_X, train_Y, cv=10,n_jobs=2)\n",
    "        predicted_ratings = cross_val_predict(clf_, test_X,test_Y, cv=10,n_jobs=2)\n",
    "        print scores\n",
    "        train_mean_score=scores.mean()\n",
    "        print \"Testing validations evaluations for : \"+str(clf_names[i])\n",
    "\n",
    "\n",
    "        r2 = r2_score(test_Y, predicted_ratings, multioutput='uniform_average')\n",
    "        mae = median_absolute_error(test_Y, predicted_ratings)\n",
    "        msle =mean_squared_log_error(test_Y, predicted_ratings) \n",
    "        mse = mean_squared_error(test_Y, predicted_ratings)\n",
    "        mae = mean_absolute_error(test_Y, predicted_ratings)\n",
    "        evs = explained_variance_score(test_Y, predicted_ratings)  \n",
    "        \n",
    "        data ={'training score':train_mean_score,\n",
    "            'r2 score':r2,\n",
    "            'median absolute error':mae,\n",
    "            'mean squared log error':msle,\n",
    "            'mean sqaured error':mse,\n",
    "            'mean absolute error':mae,\n",
    "            'explained variance error':evs}\n",
    "        \n",
    "        \n",
    "        ClassifierEvaluationResults[clf_names[i]] = data\n",
    "        #print data\n",
    "    \n",
    "    ModelEvaluationResults[model_names[m]]=ClassifierEvaluationResults\n",
    "    print ClassifierEvaluationResults\n",
    "    print \"********************************************************************\"\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}