educoder平台MapReduce基础实战.docx

1、educoder平台MapReduce基础实战MapReduce第1关:成绩统计过关代码:import java、io、IOException;import java、util、StringTokenizer;import java、io、IOException;import java、util、StringTokenizer;import org、apache、hadoop、conf、Configuration;import org、apache、hadoop、fs、Path;import org、apache、hadoop、io、*;import org、apache、hadoop、io、

2、Text;import org、apache、hadoop、mapreduce、Job;import org、apache、hadoop、mapreduce、Mapper;import org、apache、hadoop、mapreduce、Reducer;import org、apache、hadoop、mapreduce、lib、input、;import org、apache、hadoop、mapreduce、lib、output、;import org、apache、hadoop、util、GenericOptionsParser;public class WordCount /* B

3、egin */ public static class TokenizerMapper extends Mapper private final static IntWritable one = new IntWritable(1); private Text word = new Text(); private int maxValue = 0; public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException StringTokenizer itr

4、= new StringTokenizer(value、toString(),n); while (itr、hasMoreTokens() String str = itr、nextToken()、split( ); String name = str0; one、set(Integer、parseInt(str1); word、set(name); context、write(word,one); /context、write(word,one); public static class IntSumReducer extends Reducer private IntWritable re

5、sult = new IntWritable(); public void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException int maxAge = 0; int age = 0; for (IntWritable intWritable : values) maxAge = Math、max(maxAge, intWritable、get(); result、set(maxAge); context、write(key, result); public st

6、atic void main(String args) throws Exception Configuration conf = new Configuration(); Job job = new Job(conf, word count); job、setJarByClass(WordCount、class); job、setMapperClass(TokenizerMapper、class); job、setbinerClass(IntSumReducer、class); job、setReducerClass(IntSumReducer、class); job、setOutputKe

7、yClass(Text、class); job、setOutputValueClass(IntWritable、class); String inputfile = /user/test/input; String outputFile = /user/test/output/; (job, new Path(inputfile); (job, new Path(outputFile); job、waitForpletion(true); /* End */ 命令行touch file01echo Hello World Bye Worldcat file01echo Hello World

8、Bye World file01cat file01touch file02echo Hello Hadoop Goodbye Hadoop file02cat file02start-dfs、shhadoop fs -mkdir /usrhadoop fs -mkdir /usr/inputhadoop fs -ls /usr/outputhadoop fs -ls /hadoop fs -ls /usrhadoop fs -put file01 /usr/inputhadoop fs -put file02 /usr/inputhadoop fs -ls /usr/input测评MapRe

9、duce第2关:文件内容合并去重代码import java、io、IOException;import java、util、*;import org、apache、hadoop、conf、Configuration;import org、apache、hadoop、fs、Path;import org、apache、hadoop、io、*;import org、apache、hadoop、mapreduce、Job;import org、apache、hadoop、mapreduce、Mapper;import org、apache、hadoop、mapreduce、Reducer;impor

10、t org、apache、hadoop、mapreduce、lib、input、;import org、apache、hadoop、mapreduce、lib、output、;import org、apache、hadoop、util、GenericOptionsParser;public class Merge /* * param args * 对A,B两个文件进行合并,并剔除其中重复得内容,得到一个新得输出文件C */ /在这重载map函数,直接将输入中得value复制到输出数据得key上 注意在map方法中要抛出异常:throws IOException,InterruptedExce

11、ption /* Begin */ public static class Map extends Mapper protected void map(LongWritable key, Text value, Mapper、Context context) throws IOException, InterruptedException String str = value、toString(); String data = str、split( ); Text t1= new Text(data0); Text t2 = new Text(data1); context、write(t1,

12、t2); /* End */ /在这重载reduce函数,直接将输入中得key复制到输出数据得key上 注意在reduce方法上要抛出异常:throws IOException,InterruptedException /* Begin */ public static class Reduce extends Reducer protected void reduce(Text key, Iterable values, Reducer、Context context) throws IOException, InterruptedException List list = new Arra

13、yList(); for (Text text : values) String str = text、toString(); if(!list、contains(str) list、add(str); Collections、sort(list); for (String text : list) context、write(key, new Text(text); /* End */ public static void main(String args) throws Exception Configuration conf = new Configuration(); Job job

14、= new Job(conf, word count); job、setJarByClass(Merge、class); job、setMapperClass(Map、class); job、setbinerClass(Reduce、class); job、setReducerClass(Reduce、class); job、setOutputKeyClass(Text、class); job、setOutputValueClass(Text、class); String inputPath = /user/tmp/input/; /在这里设置输入路径 String outputPath =

15、/user/tmp/output/; /在这里设置输出路径 (job, new Path(inputPath); (job, new Path(outputPath); System、exit(job、waitForpletion(true) ? 0 : 1); 测评MapReduce第3关:信息挖掘 - 挖掘父子关系代码import java、io、IOException;import java、util、*;import org、apache、hadoop、conf、Configuration;import org、apache、hadoop、fs、Path;import org、apac

16、he、hadoop、io、IntWritable;import org、apache、hadoop、io、Text;import org、apache、hadoop、mapreduce、Job;import org、apache、hadoop、mapreduce、Mapper;import org、apache、hadoop、mapreduce、Reducer;import org、apache、hadoop、mapreduce、lib、input、;import org、apache、hadoop、mapreduce、lib、output、;import org、apache、hadoop、

17、util、GenericOptionsParser;public class simple_data_mining public static int time = 0; /* * param args * 输入一个child-parent得表格 * 输出一个体现grandchild-grandparent关系得表格 */ /Map将输入文件按照空格分割成child与parent,然后正序输出一次作为右表,反序输出一次作为左表,需要注意得就是在输出得value中必须加上左右表区别标志 public static class Map extends Mapper public void map(

18、Object key, Text value, Context context) throws IOException,InterruptedException/* Begin */ String child_name = new String(); String parent_name = new String(); String relation_type = new String(); String line = value、toString(); int i = 0; while(line、charAt(i) != ) i+; String values = line、substrin

19、g(0,i),line、substring(i+1); if(values0、pareTo(child) != 0) child_name = values0; parent_name = values1; relation_type = 1;/左右表区分标志 context、write(new Text(values1), new Text(relation_type+child_name+parent_name); /左表 relation_type = 2; context、write(new Text(values0), new Text(relation_type+child_nam

20、e+parent_name); /右表 /* End */ public static class Reduce extends Reducer public void reduce(Text key, Iterable values,Context context) throws IOException,InterruptedException /* Begin */ if(time = 0) /输出表头 context、write(new Text(grand_child), new Text(grand_parent); time+; int grand_child_num = 0; S

21、tring grand_child = new String10; int grand_parent_num = 0; String grand_parent= new String10; Iterator ite = values、iterator(); while(ite、hasNext() String record = ite、next()、toString(); int len = record、length(); int i = 2; if(len = 0) continue; char relation_type = record、charAt(0); String child_

22、name = new String(); String parent_name = new String(); /获取value-list中value得child while(record、charAt(i) != +) child_name = child_name + record、charAt(i); i+; i=i+1; /获取value-list中value得parent while(ilen) parent_name = parent_name+record、charAt(i); i+; /左表,取出child放入grand_child if(relation_type = 1)

23、grand_childgrand_child_num = child_name; grand_child_num+; else/右表,取出parent放入grand_parent grand_parentgrand_parent_num = parent_name; grand_parent_num+; if(grand_parent_num != 0 & grand_child_num != 0 ) for(int m = 0;mgrand_child_num;m+) for(int n=0;ngrand_parent_num;n+) context、write(new Text(grand

24、_childm), new Text(grand_parentn); /输出结果 /* End */ public static void main(String args) throws Exception / TODO Auto-generated method stub Configuration conf = new Configuration(); Job job = Job、getInstance(conf,Single table join); job、setJarByClass(simple_data_mining、class); job、setMapperClass(Map、

25、class); job、setReducerClass(Reduce、class); job、setOutputKeyClass(Text、class); job、setOutputValueClass(Text、class); String inputPath = /user/reduce/input; /设置输入路径 String outputPath = /user/reduce/output; /设置输出路径 (job, new Path(inputPath); (job, new Path(outputPath); System、exit(job、waitForpletion(true) ? 0 : 1); 测评