当前位置:   article > 正文

Flink开发环境搭建及WordCount_flink edit streamwordcount configuration 怎么配置

flink edit streamwordcount configuration 怎么配置
  1. <?xml version="1.0" encoding="UTF-8"?>
  2. <project xmlns="http://maven.apache.org/POM/4.0.0"
  3. xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  4. xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  5. <parent>
  6. <artifactId>bigdata16</artifactId>
  7. <groupId>com.shujia</groupId>
  8. <version>1.0-SNAPSHOT</version>
  9. </parent>
  10. <modelVersion>4.0.0</modelVersion>
  11. <artifactId>Flink</artifactId>
  12. <version>1.0</version>
  13. <properties>
  14. <maven.compiler.source>8</maven.compiler.source>
  15. <maven.compiler.target>8</maven.compiler.target>
  16. <flink.version>1.11.2</flink.version>
  17. <scala.binary.version>2.11</scala.binary.version>
  18. <scala.version>2.11.12</scala.version>
  19. <log4j.version>2.12.1</log4j.version>
  20. </properties>
  21. <dependencies>
  22. <dependency>
  23. <groupId>org.apache.flink</groupId>
  24. <artifactId>flink-walkthrough-common_${scala.binary.version}</artifactId>
  25. <version>${flink.version}</version>
  26. </dependency>
  27. <dependency>
  28. <groupId>org.apache.flink</groupId>
  29. <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
  30. <version>${flink.version}</version>
  31. </dependency>
  32. <dependency>
  33. <groupId>org.apache.flink</groupId>
  34. <artifactId>flink-clients_${scala.binary.version}</artifactId>
  35. <version>${flink.version}</version>
  36. </dependency>
  37. <dependency>
  38. <groupId>org.apache.logging.log4j</groupId>
  39. <artifactId>log4j-slf4j-impl</artifactId>
  40. <version>${log4j.version}</version>
  41. </dependency>
  42. <dependency>
  43. <groupId>org.apache.logging.log4j</groupId>
  44. <artifactId>log4j-api</artifactId>
  45. <version>${log4j.version}</version>
  46. </dependency>
  47. <dependency>
  48. <groupId>org.apache.logging.log4j</groupId>
  49. <artifactId>log4j-core</artifactId>
  50. <version>${log4j.version}</version>
  51. </dependency>
  52. </dependencies>
  53. <build>
  54. <plugins>
  55. <!-- Java Compiler -->
  56. <plugin>
  57. <groupId>org.apache.maven.plugins</groupId>
  58. <artifactId>maven-compiler-plugin</artifactId>
  59. <version>3.1</version>
  60. <configuration>
  61. <source>1.8</source>
  62. <target>1.8</target>
  63. </configuration>
  64. </plugin>
  65. <!-- Scala Compiler -->
  66. <plugin>
  67. <groupId>org.scala-tools</groupId>
  68. <artifactId>maven-scala-plugin</artifactId>
  69. <version>2.15.2</version>
  70. <executions>
  71. <execution>
  72. <goals>
  73. <goal>compile</goal>
  74. <goal>testCompile</goal>
  75. </goals>
  76. </execution>
  77. </executions>
  78. </plugin>
  79. </plugins>
  80. </build>
  81. </project>
  1. package com.shujia.core
  2. import org.apache.flink.streaming.api.scala._
  3. object Demo01WordCount {
  4. def main(args: Array[String]): Unit = {
  5. /**
  6. * 创建Flink入口
  7. */
  8. val env: StreamExecutionEnvironment= StreamExecutionEnvironment.getExecutionEnvironment
  9. //默认并行度等于CPU的逻辑核数 相当于是任务的一个并行度
  10. env.setParallelism(2)
  11. /**
  12. * 通过Socket模拟实时数据
  13. * nc -lk 8888
  14. *
  15. * DataStream:Flink中的编程模型
  16. */
  17. val linesDS: DataStream[String] = env.socketTextStream("master", 8888)
  18. //对每一条数据进行切分
  19. val wordsDS: DataStream[String] = linesDS
  20. .flatMap(line => line.split(","))
  21. //将每个单词变成 K V 格式
  22. val wordsKVDS: DataStream[(String, Int)] = wordsDS.map(word => (word, 1))
  23. //按照每个单词进行分组
  24. val keyByDS: KeyedStream[(String, Int), String] = wordsKVDS.keyBy(kv => kv._1)
  25. //统计每个单词的数量
  26. val wordCntDS: DataStream[(String, Int)] = keyByDS.sum(1)
  27. //将结果打印
  28. wordCntDS.print()
  29. //启动任务
  30. env.execute("Demo01WordCount")
  31. }
  32. }

声明:本文内容由网友自发贡献,不代表【wpsshop博客】立场,版权归原作者所有,本站不承担相应法律责任。如您发现有侵权的内容,请联系我们。转载请注明出处:https://www.wpsshop.cn/w/笔触狂放9/article/detail/598002
推荐阅读
相关标签
  

闽ICP备14008679号