Java實現MapReduce Wordcount案例

先改pom.xml：

<project xmlns="http://maven.apache.org/POM/4.0.0"
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>com.mcq</groupId>
	<artifactId>mr-1101</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<dependencies>
		<dependency>
			<groupId>jdk.tools</groupId>
			<artifactId>jdk.tools</artifactId>
			<version>1.8</version>
			<scope>system</scope>
			<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
		</dependency>
		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>RELEASE</version>
		</dependency>
		<dependency>
			<groupId>org.apache.logging.log4j</groupId>
			<artifactId>log4j-core</artifactId>
			<version>2.8.2</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-common</artifactId>
			<version>2.7.2</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-client</artifactId>
			<version>2.7.2</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-hdfs</artifactId>
			<version>2.7.2</version>
		</dependency>
	</dependencies>
</project>

在resources文件夾下添加文件 log4j.properties：

log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

 WordcountDriver.java：

package com.mcq;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class WordcountDriver{
	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		System.out.println("hello");
		Configuration conf=new Configuration();
		//1.獲取Job對象
		Job job=Job.getInstance(conf);
		//2.設置jar存儲位置
		job.setJarByClass(WordcountDriver.class);
		//3.關聯Map和Reduce類
		job.setMapperClass(WordcountMapper.class);
		job.setReducerClass(WordcountReducer.class);
		//4.設置Mapper階段輸出數據的key和value類型
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(IntWritable.class);
		//5.設置最終輸出的key和value類型
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);
		//6.設置輸入路徑和輸出路徑
		FileInputFormat.setInputPaths(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));
		//7.提交Job
//		job.submit();
		job.waitForCompletion(true);
//		boolean res=job.waitForCompletion(true);//true表示列印結果
//		System.exit(res?0:1);
	}
}

 WordcountMapper.java：

package com.mcq;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

//map階段
//KEYIN:輸入數據的key（偏移量，比如第一行是0~19，第二行是20~25），必須是LongWritable
//VALUEIN:輸入數據的value（比如文本內容是字元串，那就填Text）
//KEYOUT:輸出數據的key類型
//VALUEOUT:輸出數據的值類型
public class WordcountMapper extends Mapper<LongWritable, Text, Text, IntWritable>{
	IntWritable v=new IntWritable(1);
	Text k = new Text();
	@Override
	protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, IntWritable>.Context context)
			throws IOException, InterruptedException {
		// TODO Auto-generated method stub
		//1.獲取一行
		String line=value.toString();
		//2.切割單詞
		String[] words=line.split(" ");
		//3.迴圈寫出
		for(String word:words) {
			k.set(word);
			context.write(k, v);
		}
	}
}

 WordcountReducer.java：

package com.mcq;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

//KEYIN、VALUEIN：map階段輸出的key和value類型
public class WordcountReducer extends Reducer<Text, IntWritable, Text, IntWritable>{
	IntWritable v=new IntWritable();
	@Override
	protected void reduce(Text key, Iterable<IntWritable> values,
			Reducer<Text, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException {
		// TODO Auto-generated method stub
		int sum=0;
		for(IntWritable value:values) {
			sum+=value.get();
		}
		v.set(sum);
		context.write(key, v);
	}
}

在run configuration裡加上參數e:/mrtest/in.txt e:/mrtest/out.txt

運行時遇到了個bug，參考https://blog.csdn.net/qq_40310148/article/details/86617512解決了

在集群上運行：

用maven打成jar包，需要添加一些打包依賴：

	<build>
		<plugins>
			<plugin>
				<artifactId>maven-compiler-plugin</artifactId>
				<version>2.3.2</version>
				<configuration>
					<source>1.8</source>
					<target>1.8</target>
				</configuration>
			</plugin>
			<plugin>
				<artifactId>maven-assembly-plugin </artifactId>
				<configuration>
					<descriptorRefs>
						<descriptorRef>jar-with-dependencies</descriptorRef>
					</descriptorRefs>
					<archive>
						<manifest>
							<mainClass>com.mcq.WordcountDriver</mainClass>
						</manifest>
					</archive>
				</configuration>
				<executions>
					<execution>
						<id>make-assembly</id>
						<phase>package</phase>
						<goals>
							<goal>single</goal>
						</goals>
					</execution>
				</executions>
			</plugin>
		</plugins>
	</build>

 註意上面mainClass里要填驅動類的主類名，可以點擊類名右鍵copy qualified name。

將程式打成jar包（具體操作：右鍵工程名run as maven install，然後target文件夾會產生兩個jar包，我們把不用依賴的包拷貝到hadoop集群上，因為集群已經配好相關依賴了），上傳到集群

輸入以下命令運行

hadoop jar mr-1101-0.0.1-SNAPSHOT.jar com.mcq.WordcountDriver /xiaocao.txt /output

註意這裡輸入輸出的路徑是集群上的路徑。