基于hadoop的电影推荐系统的实现

2023-09-16 00:05| 来源: 网络整理| 查看: 265

1.设计任务

通过编写代码，设计一个基于Hadoop的电影推荐系统，通过此推荐系统的编写，掌握在Hadoop平台上的文件操作，数据处理的技能。

工程文件放在百度网盘了，运行run.py即可启动程序，由于代码年份久远，我已尽量打了注释，大家可以下载后进行摸索。

链接：https://pan.baidu.com/s/17OpSNstnFA1nVxDisuNBBg 提取码：9fv3 如果百度网盘失效请移步github，里面有详细的数据集和代码

2.开发环境

windows 10 hadoop 2.8.3 python 3.+ vscode MySQL 8.0 关于hadoop配置可以看我之前的文章 win10下配置hadoop环境

3.系统层次设计

为了使电影推荐系统的设计更加完善，提高推荐系统的可维护性和可拓展性。我们采用三种系统相结合的方式进行总的系统层次设计。分为Hadoop为主的分布式系统，python为主的算法分析系统和MySQL为主的数据库存储系统。三种系统之间相互独立，仅采用特定的程序接口进行调用，可以最大程度的增强推荐系统的稳健性。该系统的设计框架图如下所示：在这里插入图片描述

4.数据上传与下载

1.启动hadoop 在这里插入图片描述 2.将ml-100k数据集上传到hadoop下的ml-100k文件夹上传结果如下： 3.与python相连接利用python中的hdfs包，用如下代码实现hadoop与python相连接。

from hdfs import * client = Client("http://localhost:50070")

4.数据下载

client.download(file_user_movie,"D://VSC//vscode python//hadoop//ml-100k") client.download(file_movie_info,"D://VSC//vscode python//hadoop//ml-100k") client.download(user_information,"D://VSC//vscode python//hadoop//ml-100k") 5.面向电影制作方的推荐数据

运行结果如下图所示：在这里插入图片描述总览如下：

6.基于协同过滤算法的用户推荐算法

运行结果如下图：在这里插入图片描述

7.将推荐结果存入数据库以便查阅 db = pymysql.connect(host='127.0.0.1', port=3306, user='root', passwd='123456', db='test', charset='utf8') cursor = db.cursor() #新建数据库访问游标

main函数调用

cursor.execute("drop table if exists Recommended") #检查数据库中是否存在名为recommended的数据库，若存在则删除 sql = """ create table Recommended ( film varchar(80) not null, rating DECIMAL (7, 4) not null ) """ cursor.execute(sql) #创建数据库 for i in result.keys(): print("film:%s,rating:%f"%(movies[i],result[i])) sql = 'insert into Recommended (`film`,`rating`) values("%s","%f")'%(movies[i],result[i]) cursor.execute(sql) #执行数据插入语句 db.commit() #提交结果到数据库中

在这里插入图片描述

8.参考资料

[1]《大数据原理及应用》电子邮电出版社林子雨著 [2] Python如何连接mysql数据库 https://www.py.cn/jishu/jichu/12706.html [3] Movielens数据集详细介绍 https://blog.csdn.net/zwqhehe/article/details/75912003 [4] Python3调用Hadoop的API https://www.cnblogs.com/sss4/p/10443497.html [5] Collaborative Filtering(协同过滤算法详解) https://www.cnblogs.com/ECJTUACM-873284962/p/8729010.html

9.部分源码

完整代码请移步基于Hadoop的电影推荐系统下载或者我的github，里面有详细的数据集和代码

def score_analysis(): #电影评分分析 scores = {"1":0,"2":0,"3":0,"4":0,"5":0} for line in open(user_movie): user, item, score = line.split('\t')[0:3] scores[score] += 1 x1 = scores.keys() #用字典的键也就是分数作为x轴 y1 = scores.values() #用字典键对应的值也就是打分人数作为y轴 # print(scores) plt.figure(figsize=(19, 10)) plt.subplot(2,3,2) plt.bar(x1,y1,color = 'slateblue',width = 0.95) plt.title("电影评分统计图",fontsize=14) plt.xlabel("影片评分 (0-5)",fontsize=14) plt.ylabel("评分人数",fontsize = 14) def movie_year_analysis(): #电影年份分析 yearCounts = {} #用来统计电影年份与数量的对应关系 for year in range(1922,1999): #按照数据集中电影的年份信息生成年份字典 yearCounts[str(year)] = 0 for line in open(movie_information,encoding='ISO-8859-1'): release_date = line.split('|')[2] release_year = release_date[-4:] if release_year == "": continue yearCounts[release_year] += 1 x2 = list(yearCounts.keys()) #获取x轴坐标并转为列表 y2 = list(yearCounts.values())#获取y轴坐标并转为列表 plt.subplot(2,3,5) plt.plot(x2,y2,label = '电影数量',color = 'cornflowerblue') plt.legend(loc="upper right") #设置标签图在右上角 x_major_locator=MultipleLocator(5) #设置x轴间隔为5 ax=plt.gca() ax.xaxis.set_major_locator(x_major_locator) plt.xticks(rotation = -60) #将x轴坐标旋转60度 x_new = range(1922,1998) plt.title("电影年份统计图",fontsize=14) plt.xlabel("年份",fontsize=14) plt.ylabel("电影数量",fontsize = 14) # plt.show() def occuptin_analysis(): #职业分析 occuption_count = {} for line in open(user_information): occuption = line.split('|')[3] occuption_count[occuption] = occuption_count.get(occuption,0) + 1 #统计职业及其对应人数 sort_occuption_counts = sorted(occuption_count.items(),key=lambda k: k[1]) #将结果从小到大排序 # print(sort_occuption_counts) #由于排序之后，字典类型会变成列表类型 #然而这两种类型无法相互转换 #所以得遍历来将列表的值转换为字典 sort_occuption = {} for i in sort_occuption_counts: #将排序结果存回字典 sort_occuption[i[0]] = i[1] # print(sort_occuption) x4 = sort_occuption.keys() y4 = sort_occuption.values() plt.subplot(2,3,6) plt.bar(x4,y4,color = 'blue',width = 0.8) plt.xticks(rotation = -60) #将x轴坐标旋转60度 # plt.title("用户职业统计情况",fontsize=14) plt.xlabel("职业",fontsize=14) plt.ylabel("人数",fontsize=14) def gender_analysis(): #性别分析 unames = ['userid','age','gender','occupation','zip code'] users = pd.read_table('u.user',sep = '\|',names = unames) rnames = ['userid','itemid','rating','timestamp'] ratings = pd.read_table('u.data',names = rnames,engine = 'python') inames = ['itemid','movie title','release date','video release date','IMDB URL','unknown','Action','Adventure' ,'Animation','Children\'s','Comedy','Crime','DOcumentrary','Drama','Fatasy','Film-Noir','Horror','Musical' ,'Mystery','Romance','Sci-Fi','Thriller','War','Western'] items = pd.read_table('u.item',sep = '\|',names = inames,engine = 'python') #计算男女生对电影评分的平均值和标准差 users_df = pd.DataFrame() users_df['userid'] = users['userid'] users_df['gender'] = users['gender'] ratings_df = pd.DataFrame() ratings_df['userid'] = ratings['userid'] ratings_df['rating'] = ratings['rating'] rating_df = pd.merge(users_df,ratings_df) gender_table = pd.pivot_table(rating_df,index = ['gender','userid'],values = 'rating') gender_df = pd.DataFrame(gender_table) Female_df = gender_df.query("gender == ['F']") Male_df = gender_df.query("gender == ['M']") print("男性电影评分平均值："+str(Male_df.rating.sum()/len(Male_df.rating)) +" 标准差："+str(np.std(Male_df.rating))) print("女生电影评分平均值："+str(Female_df.rating.sum()/len(Female_df.rating))+"标准差："+str(np.std(Female_df.rating))) ratings_df_2 = pd.DataFrame() ratings_df_2['userid'] = ratings['userid'] ratings_df_2['rating'] = ratings['rating'] ratings_df_2['itemid'] = ratings['itemid'] items_df = pd.DataFrame() items_df['itemid'] = items['itemid'] items_df['movietitle'] = items['movie title'] tmp = pd.merge(users_df,ratings_df_2) gender_item_df = pd.merge(tmp,items_df) Female_item_df = gender_item_df.query("gender == ['F']") Male_item_df = gender_item_df.query("gender == ['M']") print("女生最爱看的五部电影") print(Female_item_df.groupby(['movietitle']).rating.mean().sort_values(ascending = False)[0:5,]) print("男生最爱看的五部电影") print(Male_item_df.groupby(['movietitle']).rating.mean().sort_values(ascending = False)[0:5,]) def gender_compare(): u_cols = ['user_id', 'age', 'sex', 'occupation', 'zip_code'] users = pd.read_csv(user_information, sep='|', names=u_cols,encoding='latin-1') r_cols = ['user_id', 'movie_id', 'rating', 'unix_timestamp'] ratings = pd.read_csv(user_movie, sep='\t', names=r_cols,encoding='latin-1') m_cols = ['movie_id', 'title', 'release_date', 'video_release_date', 'imdb_url'] movies = pd.read_csv(movie_information, sep='|', names=m_cols, usecols=range(5),encoding='latin-1') # 数据集整合 movie_ratings = pd.merge(movies, ratings) lens = pd.merge(movie_ratings, users) labels = ['0-9', '10-19', '20-29', '30-39', '40-49', '50-59', '60-69', '70-79'] lens['age_group'] = pd.cut(lens.age, range(0, 81, 10), right=False, labels=labels) lens[['age', 'age_group']].drop_duplicates()[:10] lens.groupby('age_group').agg({'rating': [np.size, np.mean]}) most_50 = lens.groupby('movie_id').size().sort_values(ascending=False)[:50] lens.set_index('movie_id', inplace=True) by_age = lens.loc[most_50.index].groupby(['title', 'age_group']) lens.reset_index('movie_id', inplace=True) pivoted = lens.pivot_table(index=['movie_id', 'title'], columns=['sex'], values='rating', fill_value=0) pivoted['diff'] = pivoted.M - pivoted.F plt.subplot(2,3,3) users.age.plot.hist(bins=30,edgecolor='black') plt.title("用户年龄分布图",fontsize=14) plt.ylabel('用户数量',fontsize=14) plt.xlabel('用户年龄',fontsize=14) pivoted.reset_index('movie_id', inplace=True) disagreements = pivoted[pivoted.movie_id.isin(most_50.index)]['diff'] plt.subplot(1,3,1) disagreements.sort_values().plot(kind='barh',color = 'dodgerblue') plt.title('男/女性平均评分\n(差异>0=受男性青睐)',fontsize=14) plt.ylabel('电影',fontsize=14) plt.xlabel('平均评级差',fontsize=14) def ItemSimilarity(user_movie): C = {} #存放最终的物品相似度矩阵 N = {} #存放每个电影评分人数 for user, items in user_movie.items(): for i in items.keys(): print(i) N.setdefault(i,0) N[i] += 1 C.setdefault(i,{}) for j in items.keys(): if i == j : continue C[i].setdefault(j,0) C[i][j] += 1 W = {} #计算最终物品余弦相似度矩阵 for i, related_items in C.items(): W.setdefault(i,{}) for j,cij in related_items.items(): W[i][j] = cij /(math.sqrt(N[i] * N[j])) return W def Recommend(user, user_movie, W, K, N): #根据矩阵为具体用户推荐 rank = {} action_item = user_movie[user] for item, score in action_item.items(): for j, wj in sorted(W[item].items(),key = lambda x: x[1], reverse = True)[0:K]: if j in action_item.keys(): continue rank.setdefault(j,0) rank[j] += score * wj return dict(sorted(rank.items(),key = lambda x:x[1],reverse= True)[0:N]) #排序取前N个结果 class information_counts(MRJob): """ 聚合单个用户的下的所有评分数据格式为：user_id, (item_count, rating_sum, [(item_id,rating)...]) """ def group_by_user_rating(self, key, line): user_id, item_id, rating = line.split('|') yield user_id, (item_id, float(rating)) def count_ratings_users_freq(self, user_id, values): item_count = 0 item_sum = 0 final = [] for item_id, rating in values: item_count += 1 item_sum += rating final.append((item_id, rating)) yield user_id, (item_count, item_sum, final) def steps(self): return [MRStep(mapper=self.group_by_user_rating, reducer=self.count_ratings_users_freq),]

【本文地址】

公司简介

联系我们