赞
踩
在工作中用spark对数据查询,所使用的常用语法进行整理归纳如下:
data = spark.sql("""select * from temp.tables""")
data.show(3)
# 转成pandas方式
# df = data.toPandas()
# df.head(3)
data = spark.sql("""select * from temp.tables""").distinct()
data.show(3)
#or
data = spark.sql("""select * from temp.tables""")
data = data.distinct()
data.show(3)
data = spark.sql("""select * from temp.tables""")
data.count()
data = data.select('dt', 'order_money')
data.show(3)
data = spark.sql("""select * from temp.tables""")
df = data.group('dt').agg(fn.countDistinct('user_id'), fn.sum('order_money')).toDF('dt', 'user_uv_count', 'order_money_sum')
df.show(3)
df = df.withColumn('avg_money', (df.order_money_sum / df.user_uv_count).cast('decimal(14,4)'))
df.show(3)
df.printSchema()
data = spark.sql("""select * from temp.tables""")
data.dropDuplicates(['city']).show()
data = spark.sql("""select * from temp.tables""").limit(10)
data.show(3)
data = spark.sql("""select * from temp.tables""")
data.collect().show(3)
data = spark.sql("""select * from temp.tables""").head(5)
data.show(3)
data = spark.sql("""select * from temp.tables""").take(5)
data.show(3)
data = spark.sql("""select * from temp.tables""")
data = data.sample(fraction=0.5)
data.show(3)
data = spark.sql("""select * from temp.tables""")
data = data.select('dt', 'order_money')
data.selectExpr('dt as date', 'coalesce(order_money, 4)')
data.show(3)
data = spark.sql("""select * from temp.tables""")
data2 = spark.sql("""select * from temp.tables2""")
union_data = data.unionByName(data2)
union_data.show(6)
data = spark.sql("""select * from temp.tables""")
data.describe().show()
比describe多个四分位数(25%、50%、75%)
data = spark.sql("""select * from temp.tables""")
data.summary().show()
Copyright © 2003-2013 www.wpsshop.cn 版权所有,并保留所有权利。