赞
踩
第1关:WordCount - 词频统计
# -*- coding: UTF-8 -*- from pyspark import SparkContext if __name__ == "__main__": """ 需求:对本地文件系统URI为:/root/wordcount.txt 的内容进行词频统计 """ # ********** Begin **********# sc = SparkContext("local","pySpark") rdd = sc.textFile("/root/wordcount.txt") values = rdd.flatMap(lambda x:str(x).split(" ")).map(lambda x:(x,1)).reduceByKey(lambda x,y:x+y).sortBy(lambda x:tuple(x)[1],False) print(values.collect()) # ********** End **********#
第2关:Friend Recommendation - 好友推荐
# -*- coding: UTF-8 -*- from pyspark import SparkContext def word_couple(word1, word2): if hash(word1) > hash(word2): return word1 + '_' + word2 return word2 + '_' + word1 def relations(items): result = [] for i in range(1, len(items)): result.append((word_couple(items[0], items[i]), 0)) for j in range(i+1, len(items)): result.append((word_couple(items[i], items[j]), 1)) return result def fun2(x): values = tuple(x[1]) return ((x[0], 0) if min(values)==0 else (x[0], sum(values))) if __name__ == "__main__": """ 需求:对本地文件系统URI为:/root/friend.txt 的数据统计间接好友的数量 """ # ********** Begin **********# sc = SparkContext("local", "friend recommendation") src = sc.textFile("/root/friend.txt").map(lambda x:x.strip().encode('utf-8').split(" ")) rdd = src.flatMap(relations).reduceByKey(lambda x,y:0 if x==0 or y==0 else x+y).filter(lambda x:x[1]>0) print(rdd.collect()) # ********** End **********#
Copyright © 2003-2013 www.wpsshop.cn 版权所有,并保留所有权利。