驾驶员等级
public class abdDriver {
public static void main(String arg[]) {
try {
Path input = new Path("amazon.txt");
Path output = new Path("./outPutData");
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(abdDriver.class);
job.setMapperClass(abdMapper.class);
job.setMapOutputKeyClass(LongWritable.class);
job.setMapOutputValueClass(Text.class);
job.setReducerClass(abdReducer.class);
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(Text.class);
job.setInputFormatClass(abdRecordReader.class);
//job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job, input);
FileOutputFormat.setOutputPath(job, output);
output.getFileSystem(job.getConfiguration()).delete(output, true);
job.waitForCompletion(true);
System.out.println("Completed successfully");
} catch (Exception e) {
System.out.println(e.getMessage());
}
}
}
Map类
public class abdMapper extends Mapper<LongWritable, abdRecordClass, LongWritable, Text> {
@Override
public void map(LongWritable key, abdRecordClass value, Context con) throws IOException, InterruptedException {
if (value.getTITLE().toString().trim().length() > 0) {
con.write(key, value.getTITLE());
System.out.println(key + "\t" + value.getTITLE() + "\t" + value.getCATEGORIES());
}
}
}
减速器等级
public class abdReducer extends Reducer<LongWritable, Text, NullWritable, Text> {
@Override
public void reduce(LongWritable key, Iterable<Text> value, Context con) throws IOException, InterruptedException {
System.out.println("reducer");
for (Text t : value) {
System.out.println(t);
con.write(NullWritable.get(), t);
}
}
}
自定义可写类
public class abdRecordClass implements WritableComparable<abdRecordClass> {
private Text ID, ASIN, TITLE, GROUP, SALESRANK, REV_SUMMARY,
SIMILAR, CATEGORIES, REVIEWS;
public abdRecordClass() {
ID = new Text();
ASIN = new Text();
TITLE = new Text();
GROUP = new Text();
SALESRANK = new Text();
REV_SUMMARY = new Text();
SIMILAR = new Text();
CATEGORIES = new Text();
REVIEWS = new Text();
;
}
public abdRecordClass(Text id, Text asin, Text title, Text group, Text rank, Text similar, Text cat, Text rev_summary, Text reviews) {
super();
this.ID = id;
this.ASIN = asin;
this.TITLE = title;
this.GROUP = group;
this.SALESRANK = rank;
this.SIMILAR = similar;
this.CATEGORIES = cat;
this.REV_SUMMARY = rev_summary;
this.REVIEWS = reviews;
System.out.println("new record created");
}
@Override
public void readFields(DataInput in) throws IOException {
System.out.println("read fields");
this.ID.readFields(in);
this.ASIN.readFields(in);
this.TITLE.readFields(in);
this.GROUP.readFields(in);
this.SALESRANK.readFields(in);
this.SIMILAR.readFields(in);
this.CATEGORIES.readFields(in);
this.REV_SUMMARY.readFields(in);
this.REVIEWS.readFields(in);
}
@Override
public void write(DataOutput out) throws IOException {
this.ID.write(out);
System.out.println("write fields");
this.ASIN.write(out);
this.TITLE.write(out);
this.GROUP.write(out);
this.SALESRANK.write(out);
this.SIMILAR.write(out);
this.CATEGORIES.write(out);
this.REV_SUMMARY.write(out);
this.REVIEWS.write(out);
}
}
问题是:代码在mapper上运行良好,因为mapper中的sysout提供了完美的输出,但在mapper reducer之后,我们永远不会被调用。可写类有什么问题吗?
暂无答案!
目前还没有任何答案,快来回答吧!