一个MapReduce简单程序示例 MapReduce hadoop
代码片段和文件信息
属性 大小 日期 时间 名称
----------- --------- ---------- ----- ----
文件 1472000 2012-10-23 16:05 一个MapReduce简单程序示例.ppt
文件 8799 2012-10-22 12:46 SencondInvertIndexWithCombiner.java
----------- --------- ---------- ----- ----
1480799 2
package cn.buaa.secondInvertindex;
import java.io.IOException;
import java.util.ArrayList;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
public class SencondInvertIndexWithCombiner {
public static class InvertIndexWithCombinerMap extends Mapper {
private static final String TAG=“--InvertIndexWithCombinerMap--“;
private Text valueInfo=new Text();
private Text keyInfo=new Text();
private FileSplit spliteName;
protected void map(LongWritable key Text value
Context context)
throws IOException InterruptedException {
spliteName=(FileSplit)context.getInputSplit();
String spliteFullName=spliteName.getPath().toString();
// System.out.println(spliteFullName);
int beginIndex=spliteFullName.lastIndexOf(‘/‘);
// int endIndex=spliteFullName.lastIndexOf(‘:‘);
// System.out.println(“beginloc: “+beginIndex);
String fileName=spliteName.getPath().toString().substring(beginIndex+1);
// System.out.println(fileName);
StringTokenizer st=new StringTokenizer(value.toString());
while(st.hasMoreTokens()){
String str=st.nextToken();
// keyInfo.set(str+“:“+spliteName.getPath().toString());
//Test demo
// if(“including“.equalsIgnoreCase(str)){
// System.out.println(“including loc: “+fileName);
// }
//end of Test demo
keyInfo.set(str+“:“+fileName);
valueInfo.set(“1“);
context.write(keyInfo valueInfo);
}
}
}
public static class InvertIndexWithCombinerConbiner extends Reducer{
private int interValue;
private Text keyInfo=new Text();
private Text valueInfo=new Text();
protected void reduce(Text key Iterable value
Context context)
throws IOException InterruptedException {
int sum=0;
for (Text text : value) {
sum+=Integer.parseInt(text.toString());
}
interValue=key.toString().indexOf(“:“);
keyInfo.set(key.toString().substring(0 interValue));
valueInfo.set(key.toString().substring(interValue+1)+“:“+sum);
context.write(keyInfo valueInfo);
}
}
/* public static class InvertIndexWithCombinerPartition extends HashPartitioner{
@Override
public int getPartition(Text key Text value int numReduceTasks) {
// TODO Auto-generated method stub
return super.getPartition(key value
属性 大小 日期 时间 名称
----------- --------- ---------- ----- ----
文件 1472000 2012-10-23 16:05 一个MapReduce简单程序示例.ppt
文件 8799 2012-10-22 12:46 SencondInvertIndexWithCombiner.java
----------- --------- ---------- ----- ----
1480799 2
版权声明:本文内容由互联网用户自发贡献,该文观点仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌抄袭侵权/违法违规的内容, 请发送邮件举报,一经查实,本站将立刻删除。
评论列表(条)