基于Spark MLlib+ALS协同过滤算法实现的电影推荐系统，采用MovieLens数据集进行分析建模_用als算法完成数据建模

12,[(u'Anatomy (Anatomie) (2000)', 7.460098354696925), (u'Caveman (1981)', 7.460098354696925), (u'Prisoner of the Mountains (Kavkazsky plennik) (1996)', 7.460098354696925), (u'Grass Is Greener, The (1960)', 7.460098354696925), (u'Two Ninas (1999)', 7.460098354696925), (u'Storefront Hitchcock (1997)', 7.460098354696925), (u'Game Plan, The (2007)', 6.851993756650927), (u'Am Ende eiens viel zu kurzen Tages (Death of a superhero) (2011)', 5.953119755862417), (u'Maelstr\xf6m (2000)', 5.043983016906566), (u"Taste of Cherry (Ta'm e guilass) (1997)", 5.043983016906566)]
13,[(u'Anatomy (Anatomie) (2000)', 9.717292761785984), (u'Caveman (1981)', 9.717292761785984), (u'Prisoner of the Mountains (Kavkazsky plennik) (1996)', 9.717292761785984), (u'Grass Is Greener, The (1960)', 9.717292761785984), (u'Two Ninas (1999)', 9.717292761785984), (u'Storefront Hitchcock (1997)', 9.717292761785984), (u'Game Plan, The (2007)', 8.92519457111257), (u'Am Ende eiens viel zu kurzen Tages (Death of a superhero) (2011)', 7.7543491738638295), (u'Maelstr\xf6m (2000)', 6.570135852149747), (u"Taste of Cherry (Ta'm e guilass) (1997)", 6.570135852149747)]
14,[(u'Anatomy (Anatomie) (2000)', 9.058698884794694), (u'Caveman (1981)', 9.058698884794694), (u'Prisoner of the Mountains (Kavkazsky plennik) (1996)', 9.058698884794694), (u'Grass Is Greener, The (1960)', 9.058698884794694), (u'Two Ninas (1999)', 9.058698884794694), (u'Storefront Hitchcock (1997)', 9.058698884794694), (u'Game Plan, The (2007)', 8.320285504401454), (u'Am Ende eiens viel zu kurzen Tages (Death of a superhero) (2011)', 7.2287946792989715), (u'Maelstr\xf6m (2000)', 6.124841946809852), (u"Taste of Cherry (Ta'm e guilass) (1997)", 6.124841946809852)]
45,[(u'Anatomy (Anatomie) (2000)', 9.230162364777584), (u'Caveman (1981)', 9.230162364777584), (u'Prisoner of the Mountains (Kavkazsky plennik) (1996)', 9.230162364777584), (u'Grass Is Greener, The (1960)', 9.230162364777584), (u'Two Ninas (1999)', 9.230162364777584), (u'Storefront Hitchcock (1997)', 9.230162364777584), (u'Game Plan, The (2007)', 8.477772261073596), (u'Am Ende eiens viel zu kurzen Tages (Death of a superhero) (2011)', 7.365621646124737), (u'Maelstr\xf6m (2000)', 6.240773244218133), (u"Taste of Cherry (Ta'm e guilass) (1997)", 6.240773244218133)]
46,[(u'Anatomy (Anatomie) (2000)', 13.75480763037831), (u'Caveman (1981)', 13.75480763037831), (u'Prisoner of the Mountains (Kavkazsky plennik) (1996)', 13.75480763037831), (u'Grass Is Greener, The (1960)', 13.75480763037831), (u'Two Ninas (1999)', 13.75480763037831), (u'Storefront Hitchcock (1997)', 13.75480763037831), (u'Game Plan, The (2007)', 12.633594294095019), (u'Am Ende eiens viel zu kurzen Tages (Death of a superhero) (2011)', 10.976265077113567), (u'Maelstr\xf6m (2000)', 9.300013590941944), (u"Taste of Cherry (Ta'm e guilass) (1997)", 9.300013590941944)]
47,[(u'Anatomy (Anatomie) (2000)', 11.416261420944465), (u'Caveman (1981)', 11.416261420944465), (u'Prisoner of the Mountains (Kavkazsky plennik) (1996)', 11.416261420944465), (u'Grass Is Greener, The (1960)', 11.416261420944465), (u'Two Ninas (1999)', 11.416261420944465), (u'Storefront Hitchcock (1997)', 11.416261420944465), (u'Game Plan, The (2007)', 10.485673011449762), (u'Am Ende eiens viel zu kurzen Tages (Death of a superhero) (2011)', 9.110117343201637), (u'Maelstr\xf6m (2000)', 7.718856506436623), (u"Taste of Cherry (Ta'm e guilass) (1997)", 7.718856506436623)]
89,[(u'Anatomy (Anatomie) (2000)', 12.59224858594871), (u'Caveman (1981)', 12.59224858594871), (u'Prisoner of the Mountains (Kavkazsky plennik) (1996)', 12.59224858594871), (u'Grass Is Greener, The (1960)', 12.59224858594871), (u'Two Ninas (1999)', 12.59224858594871), (u'Storefront Hitchcock (1997)', 12.59224858594871), (u'Game Plan, The (2007)', 11.565800421222775), (u'Am Ende eiens viel zu kurzen Tages (Death of a superhero) (2011)', 10.048548995408964), (u'Maelstr\xf6m (2000)', 8.513974614316112), (u"Taste of Cherry (Ta'm e guilass) (1997)", 8.513974614316112)]
90,[(u'Anatomy (Anatomie) (2000)', 10.440899867808184), (u'Caveman (1981)', 10.440899867808184), (u'Prisoner of the Mountains (Kavkazsky plennik) (1996)', 10.440899867808184), (u'Grass Is Greener, The (1960)', 10.440899867808184), (u'Two Ninas (1999)', 10.440899867808184), (u'Storefront Hitchcock (1997)', 10.440899867808184), (u'Game Plan, The (2007)', 9.589817359847075), (u'Am Ende eiens viel zu kurzen Tages (Death of a superhero) (2011)', 8.331783887661004), (u'Maelstr\xf6m (2000)', 7.059387036269982), (u"Taste of Cherry (Ta'm e guilass) (1997)", 7.059387036269982)]
91,[(u'Anatomy (Anatomie) (2000)', 12.08746098604911), (u'Caveman (1981)', 12.08746098604911), (u'Prisoner of the Mountains (Kavkazsky plennik) (1996)', 12.08746098604911), (u'Grass Is Greener, The (1960)', 12.08746098604911), (u'Two Ninas (1999)', 12.08746098604911), (u'Storefront Hitchcock (1997)', 12.08746098604911), (u'Game Plan, The (2007)', 11.102160222596012), (u'Am Ende eiens viel zu kurzen Tages (Death of a superhero) (2011)', 9.645731111434998), (u'Maelstr\xf6m (2000)', 8.172673473234568), (u"Taste of Cherry (Ta'm e guilass) (1997)", 8.172673473234568)]
1
2
3
4
5
6
7
8
9

 部分代码展示：

# coding: utf-8
import sys
from os.path import join
 
from pyspark.sql import SparkSession
from pyspark.sql import Row
from pyspark.mllib.recommendation import ALS
 
reload(sys)
sys.setdefaultencoding("utf-8")
 
 
def train_model(training, num_iterations=10, rank=1, lambda_=0.01):
    return ALS.train(training.rdd, iterations=num_iterations, rank=rank, lambda_=lambda_, seed=0)
 
 
def tuning_model(training, test):
    testing = test.select(["userId", "movieId"]).rdd
 
    min_mse = 1e6
    best_rank = 10
    best_lambda = 1.0
    num_iterations = 10
    for rank in range(1, 5):
        for lambda_f in range(1, 5):
            lambda_ = lambda_f * 0.01
            asl = train_model(training, num_iterations, rank, lambda_)
            predictions = asl.predictAll(testing).toDF(["userId", "movieId", "p_rating"])
            rates_and_preds = test \
                .join(predictions,[test.userId == predictions.userId, test.movieId == predictions.movieId]) \
                .drop(test.userId).drop(test.movieId)
 
            MSE = rates_and_preds.rdd.map(lambda r: (r.rating - r.p_rating) ** 2) \
                      .reduce(lambda x, y: x + y)/rates_and_preds.count()
            print "rank = %s, lambda = %s, Mean Squared Error = %s" % (rank, lambda_, MSE)
 
            if MSE < min_mse:
                min_mse = MSE
                best_rank = rank
                best_lambda = lambda_
 
    print "*" * 80
    print "The best params are: rank = %s, lambda = %s, Mean Squared Error = %s" % \
          (best_rank, best_lambda, min_mse)
    print "*" * 80
    return train_model(training, num_iterations, best_rank, best_lambda)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

说明：当前文章或代码如侵犯了您的权益，请私信作者删除！