我运行的是hadoop-2.2.0,伪分布式集群。我尝试使用下面的代码来获取每个Map器和还原器所花费的时间,但是我得到的Map器和还原器的数量是0。。
JobConf conf = new JobConf(getConf(), WordCount.class);
conf.setJobName("wordcount");
// the keys are words (strings)
conf.setOutputKeyClass(Text.class);
// the values are counts (ints)
conf.setOutputValueClass(IntWritable.class);
conf.setMapperClass(MapClass.class);
conf.setCombinerClass(Reduce.class);
conf.setReducerClass(Reduce.class);
List<String> other_args = new ArrayList<String>();
for(int i=0; i < args.length; ++i) {
try {
if ("-m".equals(args[i])) {
conf.setNumMapTasks(Integer.parseInt(args[++i]));
} else if ("-r".equals(args[i])) {
conf.setNumReduceTasks(Integer.parseInt(args[++i]));
} else {
other_args.add(args[i]);
}
} catch (NumberFormatException except) {
System.out.println("ERROR: Integer expected instead of " + args[i]);
return printUsage();
} catch (ArrayIndexOutOfBoundsException except) {
System.out.println("ERROR: Required parameter missing from " +
args[i-1]);
return printUsage();
}
}
// Make sure there are exactly 2 parameters left.
if (other_args.size() != 2) {
System.out.println("ERROR: Wrong number of parameters: " +
other_args.size() + " instead of 2.");
return printUsage();
}
FileInputFormat.setInputPaths(conf, other_args.get(0));
FileOutputFormat.setOutputPath(conf, new Path(other_args.get(1)));
JobClient jobclient = new JobClient(conf);
RunningJob runjob = jobclient.submitJob(conf);
TaskReport[] maps = jobclient.getMapTaskReports(runjob.getID());
System.out.println("Number of Mappers "+maps.length);
for (TaskReport rpt : maps) {
long duration = rpt.getFinishTime() - rpt.getStartTime();
System.out.println("Mapper duration: " + duration);
}
TaskReport[] reduces = jobclient.getReduceTaskReports(runjob.getID());
System.out.println("Number of Reducers "+reduces.length);
for (TaskReport rpt : reduces) {
long duration = rpt.getFinishTime() - rpt.getStartTime();
System.out.println("Reducer duration: " + duration);
}
return 0;
我做错什么了?。。
1条答案
按热度按时间nx7onnlm1#
你就快到了。唯一的问题是,在提交的作业取得有意义的进展之前,taskreport的查询发生得太快。为了得到结果,下面的代码将执行以下操作: