-
Notifications
You must be signed in to change notification settings - Fork 1
/
recommender.py
237 lines (180 loc) · 11.9 KB
/
recommender.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
from surprise import Reader, Dataset
from surprise.model_selection import GridSearchCV
from surprise.prediction_algorithms import SVD
import pandas as pd
class Recommender():
def __init__(self, utility_matrix, reviews, rating_column, descriptor, five_feature_columns, two_group_columns):
self.utility_matrix = utility_matrix
self.dataframe = reviews
self.rating_column = rating_column
self.descriptor = descriptor
self.five_feature_columns = five_feature_columns
self.two_group_columns = two_group_columns
self.recommender_history = pd.DataFrame(columns=[self.rating_column, self.descriptor,
self.five_feature_columns[0], self.five_feature_columns[1],
self.five_feature_columns[2], self.five_feature_columns[3],
self.five_feature_columns[4],
self.two_group_columns[0], self.two_group_columns[1],
'Total Ratings', 'Average Rating',
'User Rating', 'Recommended By'])
self.product_similarity = None
self.current_user = None
def set_current_user(self, current_user):
self.current_user = current_user
return self.current_user
def clear_current_user(self):
self.current_user = None
return self.current_user
def load_content_similarity_matrix(self, file_path):
self.product_similarity = pd.read_pickle(file_path)
return self.product_similarity
# appends the first recommendation not already in the recommender history
def append_new_recommendation(self, recommendations, recommender):
new_recommendation = None
for recommendation in recommendations:
if recommendation not in self.recommender_history[self.rating_column].unique():
descriptor = self.dataframe.loc[
self.dataframe[self.rating_column] == recommendation].iloc[0][self.descriptor]
feature_one = self.dataframe.loc[
self.dataframe[self.rating_column] == recommendation].iloc[0][self.five_feature_columns[0]]
feature_two = self.dataframe.loc[
self.dataframe[self.rating_column] == recommendation].iloc[0][self.five_feature_columns[1]]
feature_three = self.dataframe.loc[
self.dataframe[self.rating_column] == recommendation].iloc[0][self.five_feature_columns[2]]
feature_four = self.dataframe.loc[
self.dataframe[self.rating_column] == recommendation].iloc[0][self.five_feature_columns[3]]
feature_five = self.dataframe.loc[
self.dataframe[self.rating_column] == recommendation].iloc[0][self.five_feature_columns[4]]
column_one = self.dataframe.loc[
self.dataframe[self.rating_column] == recommendation].iloc[0][self.two_group_columns[0]]
column_two = self.dataframe.loc[
self.dataframe[self.rating_column] == recommendation].iloc[0][self.two_group_columns[1]]
new_recommendation = {
self.rating_column: recommendation,
self.descriptor: descriptor,
self.two_group_columns[0]: column_one,
self.two_group_columns[1]: column_two,
self.five_feature_columns[0]: feature_one,
self.five_feature_columns[1]: feature_two,
self.five_feature_columns[2]: feature_three,
self.five_feature_columns[3]: feature_four,
self.five_feature_columns[4]: feature_five,
'Total Ratings': self.dataframe[self.rating_column].value_counts()[recommendation],
'Average Rating': self.dataframe.groupby([self.rating_column]).mean()['Rating'][recommendation],
'User Rating': -1,
'Recommended By': recommender}
self.recommender_history = self.recommender_history.append(new_recommendation, ignore_index=True)
break
return new_recommendation
def update_user_rating(self, user_rating):
self.recommender_history.iloc[-1, self.recommender_history.columns.get_loc('User Rating')] = user_rating
return self.recommender_history.iloc[-1][self.rating_column]
def user_favorites(self):
favorites = self.recommender_history.sort_values(by = 'User Rating', ascending = False)[self.rating_column]
return favorites
def clear_history(self):
self.recommender_history.drop(self.recommender_history.index, inplace=True)
return None
################################################################################
# UNPERSONALIZED RECOMMENDATIONS
################################################################################
# returns the URL of the product with the most ratings
def most_rated(self):
recommendations = self.dataframe[self.rating_column].value_counts().index
new_recommendation = self.append_new_recommendation(recommendations, 'Most Rated Products')
return new_recommendation
# returns a tuple with the number of ratings for the combination of group levels
def rating_by_group_levels(self, rating_column, group_levels):
df = self.dataframe[[rating_column] + list(group_levels.keys())]
for key in group_levels.keys():
df = df.loc[df[key] == group_levels[key]]
rating = list(group_levels.values()) + [df.shape[0]]
return tuple(rating)
# returns a sorted list of 9 tuples containing combinations of the top 3x3 most rated segments
def best_nine(self):
best_nine = []
top_3_first_group = list(self.dataframe[self.two_group_columns[0]].value_counts().index[0:3])
top_3_second_group = list(self.dataframe[self.two_group_columns[1]].value_counts().index[0:3])
for first in top_3_first_group:
for second in top_3_second_group:
group_levels = {self.two_group_columns[0]: first, self.two_group_columns[1]: second}
best_nine.append(self.rating_by_group_levels(self.rating_column, group_levels))
best_nine.sort(key = lambda x: x[2], reverse = True)
return best_nine
def best_one_subcategory(self):
best_one = self.best_nine()[0]
df = self.dataframe[[self.rating_column] + self.two_group_columns]
df = df.loc[df[self.two_group_columns[0]] == best_one[0]]
df = df.loc[df[self.two_group_columns[1]] == best_one[1]]
recommendations = df[self.rating_column].value_counts().index
new_recommendation = self.append_new_recommendation(recommendations, 'Best One Subcategory')
return new_recommendation
# returns the URL of the product with the most ratings in the top nine groups
def best_nine_subcategories(self):
new_recommendation = None
recommendations = []
best_nine = self.best_nine()
i = 0
while new_recommendation == None:
for best_i in best_nine:
df = self.dataframe[[self.rating_column] + self.two_group_columns]
df = df.loc[df[self.two_group_columns[0]] == best_i[0]]
df = df.loc[df[self.two_group_columns[1]] == best_i[1]]
try:
recommendations.append(df[self.rating_column].value_counts().index[i])
except:
pass
new_recommendation = self.append_new_recommendation(recommendations, 'Best Nine Subcategories')
i += 1
return new_recommendation
################################################################################
# CONTENT-BASED
################################################################################
# content-based similarity with Pearson correlation
def content_based_similarity(self):
top_favorite = self.user_favorites().iloc[0]
recommendations = self.product_similarity[top_favorite].sort_values(ascending = False)
recommendations = recommendations.drop([top_favorite], axis=0).index
new_recommendation = self.append_new_recommendation(recommendations, 'Content-Based Pearson Similarity')
return new_recommendation
################################################################################
# COLLABORATIVE FILTERING
################################################################################
# get utility matrix of current user in the recommender history
def current_utility_matrix(self):
# create local user utility matrix from recommender history
current_utility_matrix = pd.DataFrame(columns = ['User', 'URL', 'Rating'])
current_utility_matrix['URL'] = self.recommender_history['URL']
current_utility_matrix['Rating'] = self.recommender_history['User Rating']
current_utility_matrix['User'] = self.current_user
# combine local user utility matrix with global utility matrix
current_utility_matrix = current_utility_matrix.append(
self.utility_matrix[['User', 'URL', 'Rating']], ignore_index = True)
return current_utility_matrix
# grid search for matrix factorization with singular value decomposition
def grid_search_singular_value_decomposition(self, params):
# build and fit full grid search with the SVD training set
current_utility_matrix = self.current_utility_matrix()
reader = Reader(rating_scale = (1, 5))
data = Dataset.load_from_df(current_utility_matrix[['User', 'URL', 'Rating']], reader)
gs = GridSearchCV(SVD,
param_grid = params,
measures = ['rmse', 'mae'],
cv = 5)
gs.fit(data)
return (gs.best_score['rmse'], gs.best_params['rmse'])
# matrix factorization with singular value decomposition for last user in Mongo database
def singular_value_decomposition(self, n_factors, reg_all):
# build and fit full SVD training set
current_utility_matrix = self.current_utility_matrix()
reader = Reader(rating_scale = (1, 5))
data = Dataset.load_from_df(current_utility_matrix[['User', 'URL', 'Rating']], reader)
dataset = data.build_full_trainset()
algo = SVD(n_factors = n_factors, reg_all = reg_all)
algo.fit(dataset)
# calculate SVD predictions for local user
recommendations = current_utility_matrix.drop(['User', 'Rating'], axis=1).drop_duplicates()
recommendations['SVD'] = recommendations['URL'].apply(lambda x: algo.predict(self.current_user, x)[3])
recommendations = recommendations.sort_values(by = 'SVD', ascending = False)['URL']
new_recommendation = self.append_new_recommendation(recommendations, 'Singular Value Decomposition')
return new_recommendation