devbox
菜鸟追梦旅行
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

Java (Spark案例分析)_spark java

作者：菜鸟追梦旅行 | 2024-04-22 04:00:27

赞

踩

spark java

一、需求：计算网页访问量前三名

  1. import org.apache.spark.rdd.RDD
  2. import org.apache.spark.{SparkConf, SparkContext}
  3.  
  4. /**
  5.   * 需求：计算网页访问量前三名
  6.   * 用户：喜欢视频 直播
  7.   * 帮助企业做经营和决策
  8.   *
  9.   * 看数据
  10.   */
  11. object UrlCount {
  12.   def main(args: Array[String]): Unit = {
  13.     //1.加载数据
  14.     val conf:SparkConf = new SparkConf().setAppName("UrlCount").setMaster("local[2]")
  15.     //spark程序入口
  16.     val sc: SparkContext = new SparkContext(conf)
  17.     //载入数据
  18.     val rdd1: RDD[String] = sc.textFile("e:/access.log")
  19.  
  20.     //2.对数据进行计算 w,1 h,1
  21.     val rdd2: RDD[(String, Int)] = rdd1.map(line => {
  22.       val s: Array[String] = line.split("\t")
  23.       //标注为出现1次
  24.       (s(1), 1)
  25.     })
  26.  
  27.     //3.将相同的网址进行累加求和  网页，201
  28.     val rdd3:RDD[(String, Int)] = rdd2.reduceByKey(_+_)
  29.  
  30.     //4.排序 取出前三
  31.     val rdd4: Array[(String, Int)] = rdd3.sortBy(_._2, false).take(3)
  32.  
  33.     //5.遍历打印
  34.     rdd4.foreach(x => {
  35.       println("网址为：" + x._1 + "访问量为：" + x._2)
  36.     })
  37.  
  38.     //6.关闭资源
  39.     sc.stop()
  40.   }
  41. }

结果：

二、需求：求出每个学院 访问第一位的网址

  1. import java.net.URL
  2.  
  3. import org.apache.spark.rdd.RDD
  4. import org.apache.spark.{SparkConf, SparkContext}
  5.  
  6. /**
  7.   * 需求：求出每个学院 访问第一位的网址
  8.   * bigdata：video（直播）
  9.   * java：video
  10.   * python：teacher
  11.   */
  12. object UrlGroupCount {
  13.   def main(args: Array[String]): Unit = {
  14.     //1.创建sparkContext
  15.     val conf: SparkConf = new SparkConf().setAppName("UrlGroupCount").setMaster("local[2]")
  16.     val sc: SparkContext = new SparkContext(conf)
  17.  
  18.     //2.加载数据
  19.     val rdd1: RDD[String] = sc.textFile("e:/access.log")
  20.  
  21.     //3.切分
  22.     val rdd2: RDD[(String, Int)] = rdd1.map(line => {
  23.       val s: Array[String] = line.split("\t")
  24.       //网址，1
  25.       (s(1), 1)
  26.     })
  27.  
  28.     //4.求出总的访问量  网址，总的访问量
  29.     val rdd3: RDD[(String, Int)] = rdd2.reduceByKey(_+_)
  30.  
  31.     //5.取出学院
  32.     val rdd4: RDD[(String, String, Int)] = rdd3.map(x => {
  33.       //拿到url
  34.       val url: String = x._1
  35.       //java中拿到主机名
  36.       val host: String = new URL(url).getHost.split("[.]")(0)
  37.       //元组输出
  38.       (host, url, x._2)
  39.     })
  40.  
  41.     //6.按照学院进行分组
  42.     val rdd5: RDD[(String, List[(String, String, Int)])] = rdd4.groupBy(_._1).mapValues(it => {
  43.       //倒序
  44.       it.toList.sortBy(_._3).reverse.take(1)
  45.     })
  46.  
  47.     //7.遍历打印
  48.     rdd5.foreach(x => {
  49.       println("学院为：" + x._1 + "," + "访问量第一的为：" + x._2)
  50.     })
  51.  
  52.     //8.关闭资源
  53.     sc.stop()
  54.   }
  55. }

 

结果：

三、需求：加入自定义分区，按照学院分区，相同的学院分为一个结果文件

  1. import java.net.URL
  2.  
  3. import org.apache.spark.rdd.RDD
  4. import org.apache.spark.{Partitioner, SparkConf, SparkContext}
  5.  
  6. import scala.collection.mutable
  7.  
  8. /**
  9.   * 需求：加入自定义分区
  10.   * 按照学院分区，相同的学院分为一个结果文件
  11.   */
  12. object UrlParCount {
  13.   def main(args: Array[String]): Unit = {
  14.     //1.创建sparkContext
  15.     val conf: SparkConf = new SparkConf().setAppName("UrlParCount").setMaster("local[2]")
  16.     val sc: SparkContext = new SparkContext(conf)
  17.  
  18.     //2.加载数据
  19.     val rdd1 = sc.textFile("e:/access.log").map(line => {
  20.       val s: Array[String] = line.split("\t")
  21.       //元组输出
  22.       (s(1), 1)
  23.     })
  24.  
  25.     //3.聚合
  26.     val rdd2: RDD[(String, Int)] = rdd1.reduceByKey(_+_)
  27.  
  28.     //4.自定义格式
  29.     val rdd3: RDD[(String, (String, Int))] = rdd2.map(t => {
  30.       val url = t._1
  31.       val host = new URL(url).getHost
  32.       val xHost: String = host.split("[.]")(0)
  33.       //元组输出
  34.       (xHost, (url, t._2))
  35.     })
  36.  
  37.     //5.加入自定义分区
  38.     val xueyuan: Array[String] = rdd3.map(_._1).distinct().collect
  39.     val xueYuanPartitioner: XueYuanPartitioner = new XueYuanPartitioner(xueyuan)
  40.  
  41.     //6.加入分区规则
  42.     val rdd4: RDD[(String, (String, Int))] = rdd3.partitionBy(xueYuanPartitioner).mapPartitions(it => {
  43.       it.toList.sortBy(_._2._2).reverse.take(1).iterator
  44.     })
  45.  
  46.     //7.遍历打印
  47.     rdd4.saveAsTextFile("e://pout")
  48.  
  49.     //8.关闭资源
  50.     sc.stop()
  51.   }
  52. }
  53.  
  54. class XueYuanPartitioner(xy: Array[String]) extends Partitioner {
  55.   //自定义规则 学院 分区号
  56.   val rules: mutable.HashMap[String, Int] = new mutable.HashMap[String, Int]()
  57.   var number = 0
  58.  
  59.   //遍历学院
  60.   for(i <- xy){
  61.     //学院与分区号对应
  62.     rules += (i -> number)
  63.     //分区号递增
  64.     number += 1
  65.   }
  66.  
  67.   //总的分区个数
  68.   override def numPartitions: Int = xy.length
  69.  
  70.   //拿到分区
  71.   override def getPartition(key: Any): Int = {
  72.     rules.getOrElse(key.toString, 0)
  73.   }
  74. }

 

结果：

1、part-00000

(bigdata,(http://bigdata.xxxxxx.com/bigdata/video.shtml,503))

2、part-00001

(java,(http://java.xxxxxx.com/java/course/cloud.shtml,1028))

3、part-00002

(net,(http://net.xxxxxx.com/net/video.shtml,525))

四、pom.xml文件

  1. <?xml version="1.0" encoding="UTF-8"?>
  2. <project xmlns="http://maven.apache.org/POM/4.0.0"
  3.          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  4.          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  5.     <modelVersion>4.0.0</modelVersion>
  6.  
  7.     <groupId>com.demo.spark</groupId>
  8.     <artifactId>SparkWC</artifactId>
  9.     <version>1.0-SNAPSHOT</version>
  10.  
  11.     <properties>
  12.         <maven.compiler.source>1.8</maven.compiler.source>
  13.         <maven.compiler.target>1.8</maven.compiler.target>
  14.         <scala.version>2.11.8</scala.version>
  15.         <spark.version>2.2.0</spark.version>
  16.         <hadoop.version>2.8.4</hadoop.version>
  17.         <encoding>UTF-8</encoding>
  18.     </properties>
  19.  
  20.     <dependencies>
  21.         <!-- scala的依赖导入 -->
  22.         <dependency>
  23.             <groupId>org.scala-lang</groupId>
  24.             <artifactId>scala-library</artifactId>
  25.             <version>${scala.version}</version>
  26.         </dependency>
  27.  
  28.         <!-- spark的依赖导入 -->
  29.         <dependency>
  30.             <groupId>org.apache.spark</groupId>
  31.             <artifactId>spark-core_2.11</artifactId>
  32.             <version>${spark.version}</version>
  33.         </dependency>
  34.  
  35.         <!-- hadoop-client API的导入 -->
  36.         <dependency>
  37.             <groupId>org.apache.hadoop</groupId>
  38.             <artifactId>hadoop-client</artifactId>
  39.             <version>${hadoop.version}</version>
  40.         </dependency>
  41.  
  42.         <!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java -->
  43.         <dependency>
  44.             <groupId>mysql</groupId>
  45.             <artifactId>mysql-connector-java</artifactId>
  46.             <version>5.1.39</version>
  47.         </dependency>
  48.  
  49.     </dependencies>
  50.  
  51.     <build>
  52.         <pluginManagement>
  53.             <plugins>
  54.                 <!-- scala的编译插件 -->
  55.                 <plugin>
  56.                     <groupId>net.alchim31.maven</groupId>
  57.                     <artifactId>scala-maven-plugin</artifactId>
  58.                     <version>3.2.2</version>
  59.                 </plugin>
  60.                 <!-- java的编译插件 -->
  61.                 <plugin>
  62.                     <groupId>org.apache.maven.plugins</groupId>
  63.                     <artifactId>maven-compiler-plugin</artifactId>
  64.                     <version>3.5.1</version>
  65.                 </plugin>
  66.             </plugins>
  67.         </pluginManagement>
  68.         <plugins>
  69.             <plugin>
  70.                 <groupId>net.alchim31.maven</groupId>
  71.                 <artifactId>scala-maven-plugin</artifactId>
  72.                 <executions>
  73.                     <execution>
  74.                         <id>scala-compile-first</id>
  75.                         <phase>process-resources</phase>
  76.                         <goals>
  77.                             <goal>add-source</goal>
  78.                             <goal>compile</goal>
  79.                         </goals>
  80.                     </execution>
  81.                     <execution>
  82.                         <id>scala-test-compile</id>
  83.                         <phase>process-test-resources</phase>
  84.                         <goals>
  85.                             <goal>testCompile</goal>
  86.                         </goals>
  87.                     </execution>
  88.                 </executions>
  89.             </plugin>
  90.  
  91.             <plugin>
  92.                 <groupId>org.apache.maven.plugins</groupId>
  93.                 <artifactId>maven-compiler-plugin</artifactId>
  94.                 <executions>
  95.                     <execution>
  96.                         <phase>compile</phase>
  97.                         <goals>
  98.                             <goal>compile</goal>
  99.                         </goals>
  100.                     </execution>
  101.                 </executions>
  102.             </plugin>
  103.  
  104.             <!-- 打jar包插件 -->
  105.             <plugin>
  106.                 <groupId>org.apache.maven.plugins</groupId>
  107.                 <artifactId>maven-shade-plugin</artifactId>
  108.                 <version>2.4.3</version>
  109.                 <executions>
  110.                     <execution>
  111.                         <phase>package</phase>
  112.                         <goals>
  113.                             <goal>shade</goal>
  114.                         </goals>
  115.                         <configuration>
  116.                             <filters>
  117.                                 <filter>
  118.                                     <artifact>*:*</artifact>
  119.                                     <excludes>
  120.                                         <exclude>META-INF/*.SF</exclude>
  121.                                         <exclude>META-INF/*.DSA</exclude>
  122.                                         <exclude>META-INF/*.RSA</exclude>
  123.                                     </excludes>
  124.                                 </filter>
  125.                             </filters>
  126.                         </configuration>
  127.                     </execution>
  128.                 </executions>
  129.             </plugin>
  130.         </plugins>
  131.     </build>
  132.  
  133. </project>

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/菜鸟追梦旅行/article/detail/466709
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号