热门标签
热门文章
- 1pytorch实现MTCNN——代码部分(1)
- 2【LLM】本地部署Gemma模型（图文）_gemma 代码生成
- 3Diffusion 扩散模型（DDPM）详解及torch复现_def show_images(datset, num_samples=20, cols=4):
- 4设置Frame为全屏幕_frame框架怎么全屏
- 5JData算法大赛-用户购买预测_cart算法客户购买预测
- 6基于php的在线宠物电商交易平台的设计与实现_课堂作业宠物商店电子商务系统的业务逻辑如下。(1)用户注册:输入用户号、用户
- 7《书生.浦语大模型全链路开源体系》笔记_书生浦语产品架构
- 8openplc Linux 地址映射io，读写驱动数据等使用记录
- 9【NLP】什么是智能文档处理 (IDP) 以及如何开始自动化文档数据提取_nlp自动分析文本excel等数据,提取相关内容。整合为文本
- 10SpringCloud 和 Dubbo 和 Serverless 性能对比...和技术选型_serverless springcloud
当前位置: article > 正文
大数据从入门到实战 - 第3章 MapReduce基础实战
作者：很楠不爱3 | 2024-04-09 08:21:57
赞
踩
大数据从入门到实战 - 第3章 mapreduce基础实战
大数据从入门到实战 - 第3章 MapReduce基础实战
叮嘟！这里是小啊呜的学习课程资料整理。好记性不如烂笔头，今天也是努力进步的一天。一起加油进阶吧！
一、关于此次实践
1、实战简介
MapReduce是Hadoop的核心功能之一，掌握它对学习Hadoop至关重要。
Hadoop Map/Reduce是一个使用简易的软件框架，基于它写出来的应用程序能够运行在由上千个商用机器组成的大型集群上，并以一种可靠容错的方式并行处理上T级别的数据集。
本章我们来通过几个示例来学习MapReduce的用法。
2、全部任务
二、实践详解
1、第 1 关：成绩统计
import java.io.IOException; import java.util.StringTokenizer; import java.io.IOException; import java.util.StringTokenizer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.*; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.GenericOptionsParser; public class WordCount { /********** Begin **********/ //Mapper函数 public static class TokenizerMapper extends Mapper<LongWritable, Text, Text, IntWritable> { private final static IntWritable one = new IntWritable(1); private Text word = new Text(); private int maxValue = 0; public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { StringTokenizer itr = new StringTokenizer(value.toString(),"\n"); while (itr.hasMoreTokens()) { String[] str = itr.nextToken().split(" "); String name = str[0]; one.set(Integer.parseInt(str[1])); word.set(name); context.write(word,one); } //context.write(word,one); } } public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> { private IntWritable result = new IntWritable(); public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int maxAge = 0; int age = 0; for (IntWritable intWritable : values) { maxAge = Math.max(maxAge, intWritable.get()); } result.set(maxAge); context.write(key, result); } } public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = new Job(conf, "word count"); job.setJarByClass(WordCount.class); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); String inputfile = "/user/test/input"; String outputFile = "/user/test/output/"; FileInputFormat.addInputPath(job, new Path(inputfile)); FileOutputFormat.setOutputPath(job, new Path(outputFile)); job.waitForCompletion(true); /********** End **********/ } }
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
命令行 touch file01 echo Hello World Bye World cat file01 echo Hello World Bye World >file01 cat file01 touch file02 echo Hello Hadoop Goodbye Hadoop >file02 cat file02 start-dfs.sh hadoop fs -mkdir /usr hadoop fs -mkdir /usr/input hadoop fs -ls /usr/output hadoop fs -ls / hadoop fs -ls /usr hadoop fs -put file01 /usr/input hadoop fs -put file02 /usr/input hadoop fs -ls /usr/input
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
测评
2、第 2 关：文件内容合并去重
import java.io.IOException; import java.util.*; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.*; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.GenericOptionsParser; public class Merge { /** * @param args * 对A,B两个文件进行合并，并剔除其中重复的内容，得到一个新的输出文件C */ //在这重载map函数，直接将输入中的value复制到输出数据的key上 注意在map方法中要抛出异常：throws IOException,InterruptedException public static class Map extends Mapper<Object, Text, Text, Text>{ /********** Begin **********/ public void map(Object key, Text value, Context content) throws IOException, InterruptedException { Text text1 = new Text(); Text text2 = new Text(); StringTokenizer itr = new StringTokenizer(value.toString()); while (itr.hasMoreTokens()) { text1.set(itr.nextToken()); text2.set(itr.nextToken()); content.write(text1, text2); } } /********** End **********/ } //在这重载reduce函数，直接将输入中的key复制到输出数据的key上 注意在reduce方法上要抛出异常：throws IOException,InterruptedException public static class Reduce extends Reducer<Text, Text, Text, Text> { /********** Begin **********/ public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException { Set<String> set = new TreeSet<String>(); for(Text tex : values){ set.add(tex.toString()); } for(String tex : set){ context.write(key, new Text(tex)); } } /********** End **********/ } public static void main(String[] args) throws Exception{ // TODO Auto-generated method stub Configuration conf = new Configuration(); conf.set("fs.default.name","hdfs://localhost:9000"); Job job = Job.getInstance(conf,"Merge and duplicate removal"); job.setJarByClass(Merge.class); job.setMapperClass(Map.class); job.setCombinerClass(Reduce.class); job.setReducerClass(Reduce.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); String inputPath = "/user/tmp/input/"; //在这里设置输入路径 String outputPath = "/user/tmp/output/"; //在这里设置输出路径 FileInputFormat.addInputPath(job, new Path(inputPath)); FileOutputFormat.setOutputPath(job, new Path(outputPath)); System.exit(job.waitForCompletion(true) ? 0 : 1); } }
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
- 72
- 73
- 74
- 75
- 76
- 77
- 78
- 79
测评
3、第 3 关：信息挖掘 - 挖掘父子关系
import java.io.IOException; import java.util.*; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.GenericOptionsParser; public class simple_data_mining { public static int time = 0; /** * @param args * 输入一个child-parent的表格 * 输出一个体现grandchild-grandparent关系的表格 */ //Map将输入文件按照空格分割成child和parent，然后正序输出一次作为右表，反序输出一次作为左表，需要注意的是在输出的value中必须加上左右表区别标志 public static class Map extends Mapper<Object, Text, Text, Text>{ public void map(Object key, Text value, Context context) throws IOException,InterruptedException{ /********** Begin **********/ String line = value.toString(); String[] childAndParent = line.split(" "); List<String> list = new ArrayList<>(2); for (String childOrParent : childAndParent) { if (!"".equals(childOrParent)) { list.add(childOrParent); } } if (!"child".equals(list.get(0))) { String childName = list.get(0); String parentName = list.get(1); String relationType = "1"; context.write(new Text(parentName), new Text(relationType + "+" + childName + "+" + parentName)); relationType = "2"; context.write(new Text(childName), new Text(relationType + "+" + childName + "+" + parentName)); } /********** End **********/ } } public static class Reduce extends Reducer<Text, Text, Text, Text>{ public void reduce(Text key, Iterable<Text> values,Context context) throws IOException,InterruptedException{ /********** Begin **********/ //输出表头 if (time == 0) { context.write(new Text("grand_child"), new Text("grand_parent")); time++; } //获取value-list中value的child List<String> grandChild = new ArrayList<>(); //获取value-list中value的parent List<String> grandParent = new ArrayList<>(); //左表，取出child放入grand_child for (Text text : values) { String s = text.toString(); String[] relation = s.split("\\+"); String relationType = relation[0]; String childName = relation[1]; String parentName = relation[2]; if ("1".equals(relationType)) { grandChild.add(childName); } else { grandParent.add(parentName); } } //右表，取出parent放入grand_parent int grandParentNum = grandParent.size(); int grandChildNum = grandChild.size(); if (grandParentNum != 0 && grandChildNum != 0) { for (int m = 0; m < grandChildNum; m++) { for (int n = 0; n < grandParentNum; n++) { //输出结果 context.write(new Text(grandChild.get(m)), new Text( grandParent.get(n))); } } } /********** End **********/ } } public static void main(String[] args) throws Exception{ // TODO Auto-generated method stub Configuration conf = new Configuration(); Job job = Job.getInstance(conf,"Single table join"); job.setJarByClass(simple_data_mining.class); job.setMapperClass(Map.class); job.setReducerClass(Reduce.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); String inputPath = "/user/reduce/input"; //设置输入路径 String outputPath = "/user/reduce/output"; //设置输出路径 FileInputFormat.addInputPath(job, new Path(inputPath)); FileOutputFormat.setOutputPath(job, new Path(outputPath)); System.exit(job.waitForCompletion(true) ? 0 : 1); } }
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
- 47
- 48
- 49
- 50
- 51
- 52
- 53
- 54
- 55
- 56
- 57
- 58
- 59
- 60
- 61
- 62
- 63
- 64
- 65
- 66
- 67
- 68
- 69
- 70
- 71
- 72
- 73
- 74
- 75
- 76
- 77
- 78
- 79
- 80
- 81
- 82
- 83
- 84
- 85
- 86
- 87
- 88
- 89
- 90
- 91
- 92
- 93
- 94
- 95
- 96
- 97
- 98
- 99
- 100
- 101
- 102
- 103
- 104
- 105
- 106
- 107
- 108
- 109
- 110
测评
Ending！
更多课程知识学习记录随后再来吧！
就酱，嘎啦！
- 1
注：
人生在勤，不索何获。
声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/很楠不爱3/article/detail/391449
推荐阅读
相关标签
