我正在尝试执行新的api mapreduce wordcount示例。这是我的程序,我使用hadoop-core-1.0.4.jar作为插件。
import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class NewWordCount {
public static class WordMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
String[] words = line.split("\\W+");
for (String word : words) {
context.write(new Text(word), new IntWritable(1));
}
}
}
public static class WordReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
protected void reduce(Text key, Iterator<IntWritable> values,
Context context)
throws IOException, InterruptedException {
/*int sum = 0;
for (IntWritable value:values) {
sum += value.get();
}*/
int sum = 0;
while (values.hasNext()) {
sum += values.next().get();
}
context.write(key, new IntWritable(sum));
}
}
public static void main(String[] args) throws Exception
{
Configuration conf = new Configuration();
int exitCode=ToolRunner.run((Tool) new NewWordCount(), args);
System.exit(exitCode);
}
public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException
{
Path inputPath=new Path(args[0]);
Path outputPath=new Path(args[1]);
Configuration conf = new Configuration();
Job job = new Job(conf,"word count");
job.setJarByClass(NewWordCount.class);
job.setMapperClass(WordMapper.class);
job.setReducerClass(WordReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.setInputPaths(job,inputPath);
FileOutputFormat.setOutputPath(job,outputPath);
job.setNumReduceTasks(2);
// job.setPartitionerClass(VCPartitioner.class);
job.waitForCompletion(true);
//job.submit();
return 0;
}
}
在执行异常时,我得到以下错误:
Exception in thread "main" java.lang.ClassCastException: NewWordCount cannot be cast to org.apache.hadoop.util.Tool
at NewWordCount.main(NewWordCount.java:59)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.util.RunJar.main(RunJar.java:208)
有人能告诉我怎么解决这个错误吗?提前谢谢。
1条答案
按热度按时间iq3niunx1#
你不能投
NewWordCount
因为它没有实施Tool
接口。删除此行:
你只需要
main
方法(将run方法的内容复制到main之后)。要运行它: