一個典型的MapRuduce實例------webcount（網站統計訪客信息）

統計某一特定網站的某個時辰訪客人數所用版本：hadoop2.6.5 數據樣式如下：輔助類 mapper 映射特定年份中每月每天每個時辰的訪客數 reducer 彙總一個時辰內訪客人數 driver 配置信息，程式入口 command result ...

統計某一特定網站的某個時辰訪客人數

所用版本：hadoop2.6.5

數據樣式如下：

111.111.111.111 - - [16/Dec/2012:05:32:50 -0500] "GET / HTTP/1.1" 200 14791 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
111.111.111.111 - - [16/Dec/2012:05:33:50 -0500] "GET / HTTP/1.1" 200 14791 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
111.111.111.111 - - [16/Dec/2012:05:34:45 -0500] "GET / HTTP/1.1" 200 14791 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
111.111.111.111 - - [16/Dec/2012:05:34:50 -0500] "GET / HTTP/1.1" 200 14791 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
111.111.111.111 - - [16/Dec/2012:09:34:55 -0500] "GET / HTTP/1.1" 200 14791 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
111.111.111.111 - - [16/Dec/2012:10:23:30 -0500] "GET / HTTP/1.1" 200 14791 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
111.111.111.111 - - [16/Dec/2012:10:32:50 -0500] "GET / HTTP/1.1" 200 14791 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"

輔助類

 1 package com.trendwise.software;
 2 
 3 import java.text.SimpleDateFormat; 
 4 import java.util.Date; 
 5 import java.io.DataInput; import java.io.DataOutput; 
 6 import java.io.IOException; 
 7 import org.apache.hadoop.io.WritableComparable; 
 8 
 9 public class DateWritable implements WritableComparable<DateWritable>{
10     private final static SimpleDateFormat formatter = new SimpleDateFormat( "yyyy-MM-dd' T 'HH:mm:ss.SSS" ); 
11     private Date date; 
12     public Date getDate() { 
13         return date; 
14     } 
15     public void setDate( Date date ) { 
16         this.date = date; 
17     } 
18 
19     @Override
20     public void readFields(DataInput in) throws IOException {
21         date = new Date( in.readLong() );         
22     }
23 
24     @Override
25     public void write(DataOutput out) throws IOException {
26         out.writeLong( date.getTime() );         
27     }
28 
29     @Override
30     public int compareTo(DateWritable o) {
31         return date.compareTo( o.getDate() ); 
32     }
33     
34     public String toString() { 
35         return formatter.format( date); 
36     }     
37 }

mapper 映射特定年份中每月每天每個時辰的訪客數

 1 package com.trendwise.software;
 2 
 3 import java.io.IOException;
 4 import java.util.Calendar;
 5 import org.apache.hadoop.io.IntWritable;
 6 import org.apache.hadoop.io.LongWritable;
 7 import org.apache.hadoop.io.Text;
 8 import org.apache.hadoop.mapreduce.Mapper;
 9 
10 public class LogMapper extends Mapper<LongWritable, Text, DateWritable, IntWritable> { 
11     public static DateWritable dates = new DateWritable(); 
12     public final static IntWritable two = new IntWritable(1); 
13     public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { 
14         String text = value.toString(); 
15         // Get the date and time 
16         int openBracket = text.indexOf( '[' ); 
17         int closeBracket = text.indexOf( ']' ); 
18         if( openBracket != -1 && closeBracket != -1 ) { 
19             // Read the date 
20             String dateString = text.substring( text.indexOf( '[' ) + 1, text. indexOf( ']' ) ); 
21             // Build a date object from a string of the form: 16/Dec/2012:05:32:50 -0500 
22             int index = 0; 
23             int nextIndex = dateString.indexOf( '/' ); 
24             int day = Integer.parseInt( dateString.substring(index, nextIndex) );
25             
26             index = nextIndex; nextIndex = dateString.indexOf( '/', index+1 ); 
27             String month = dateString.substring( index+1, nextIndex ); 
28             index = nextIndex; 
29             nextIndex = dateString.indexOf( ':', index ); 
30             int year = Integer.parseInt(dateString.substring(index + 1, nextIndex)); 
31             index = nextIndex; nextIndex = dateString.indexOf( ':', index+1 ); 
32             int hour = Integer.parseInt(dateString.substring(index + 1, nextIndex)); 
33             // Build a calendar object for this date 
34             Calendar calendar = Calendar.getInstance(); 
35             calendar.set( Calendar.DATE, day );
36             calendar.set( Calendar.YEAR, year ); 
37             calendar.set( Calendar.HOUR, hour ); 
38             calendar.set( Calendar.MINUTE, 0 ); 
39             calendar.set( Calendar.SECOND, 0 ); 
40             calendar.set( Calendar.MILLISECOND, 0 ); 
41             if( month.equalsIgnoreCase( "dec" ) ) { 
42                 calendar.set( Calendar.MONTH, Calendar.DECEMBER ); 
43             } 
44             else if( month.equalsIgnoreCase( "nov" ) ) { 
45                 calendar.set( Calendar.MONTH, Calendar.NOVEMBER ); 
46             } 
47             else if( month.equalsIgnoreCase( "oct" ) ) { 
48                 calendar.set( Calendar.MONTH, Calendar.OCTOBER ); 
49             }
50             else if( month.equalsIgnoreCase( "sep" ) ) { 
51                 calendar.set( Calendar.MONTH, Calendar.SEPTEMBER ); 
52             } 
53             else if( month.equalsIgnoreCase( "aug" ) ) { 
54                 calendar.set( Calendar.MONTH, Calendar.AUGUST ); 
55             } 
56             else if( month.equalsIgnoreCase( "jul" ) ) { 
57                 calendar.set( Calendar.MONTH, Calendar.JULY ); 
58             } 
59             else if( month.equalsIgnoreCase( "jun" ) ) {
60                 calendar.set( Calendar.MONTH, Calendar.JUNE ); 
61             } 
62             else if( month.equalsIgnoreCase( "may" ) ) {
63                 calendar.set( Calendar.MONTH, Calendar.MAY ); 
64             } 
65             else if( month.equalsIgnoreCase( "apr" ) ) { 
66                 calendar.set( Calendar.MONTH, Calendar.APRIL ); 
67             } 
68             else if( month.equalsIgnoreCase( "mar" ) ) { 
69                 calendar.set( Calendar.MONTH, Calendar.MARCH ); 
70             } 
71             else if( month.equalsIgnoreCase( "feb" ) ) { 
72                 calendar.set( Calendar.MONTH, Calendar.FEBRUARY ); 
73             } 
74             else if( month.equalsIgnoreCase( "jan" ) ) { 
75                 calendar.set( Calendar.MONTH, Calendar.JANUARY ); 
76             } 
77             
78             dates.setDate( calendar.getTime() ); 
79             context.write(dates, two); 
80             
81         }
82     }
83 }

reducer 彙總一個時辰內訪客人數

 1 package com.trendwise.software;
 2 
 3 import java.io.IOException;
 4 import org.apache.hadoop.io.IntWritable;
 5 import org.apache.hadoop.mapreduce.Reducer;
 6  
 7 public class  LogReducer extends Reducer<DateWritable, IntWritable, DateWritable, IntWritable> {
 8     @Override
 9     public void reduce( DateWritable key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { 
10     
11         int countn = 0; 
12         for(IntWritable v :values){ 
13             countn += v.get(); 
14         }     
15         context.write(key, new IntWritable( countn) ); 
16     } 
17 }

driver 配置信息，程式入口

 1 package com.trendwise.software;
 2 
 3 import java.io.IOException;
 4 import org.apache.hadoop.conf.Configuration;
 5 import org.apache.hadoop.fs.Path;
 6 import org.apache.hadoop.io.IntWritable;
 7 import org.apache.hadoop.mapreduce.Job;
 8 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 9 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
10 
11 public class Driver {
12     
13     public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { 
14                 
15         String in = args[0];
16         String out = args[1];
17         int unitmb =Integer.valueOf(args[2]);                
18         int nreducer = Integer.valueOf(args[3]);
19         
20         Configuration conf = new Configuration();                
21         conf.set("mapreduce.input.fileinputformat.split.maxsize", String.valueOf(unitmb * 1024 * 1024));
22         conf.set("mapred.min.split.size", String.valueOf(unitmb * 1024 * 1024));
23         conf.set("mapreduce.input.fileinputformat.split.minsize.per.node", String.valueOf(unitmb * 1024 * 1024));
24         conf.set("mapreduce.input.fileinputformat.split.minsize.per.rack", String.valueOf(unitmb * 1024 * 1024));
25                 
26         Job job = new Job(conf);        
27         FileInputFormat.addInputPath(job, new Path(in));
28         FileOutputFormat.setOutputPath(job, new Path(out));            
29         job.setMapperClass(LogMapper.class); 
30         job.setReducerClass(LogReducer.class); 
31         job.setCombinerClass(LogReducer.class); 
32         job.setNumReduceTasks(nreducer);
33         job.setMapOutputKeyClass(DateWritable.class);
34         job.setMapOutputValueClass(IntWritable.class);    
35         job.setOutputKeyClass(DateWritable.class); 
36         job.setOutputValueClass(IntWritable.class);
37         job.setJarByClass(Driver.class);
38         job.waitForCompletion(true);    
39                     
40     }     
41 }

command

result