amazonhadoop2.4+avro1.77:找到了接口org.apache.hadoop.mapreduce.taskattemptcontext,但需要类

zzzyeukh  于 2021-06-02  发布在  Hadoop
关注(0)|答案(1)|浏览(372)

我试图在emr上运行以下代码,它给出了前面提到的异常。有人知道哪里出了问题吗?我正在使用avro-tools-1.77编译我的模式。
经过一番研究,我开始觉得这可能是一个avro问题,可以通过使用maven编译和编辑依赖项,或者将amazon hadoop版本更改为以前的版本来解决。但是,我从来没有使用过maven,修改hadoop版本会弄乱我的很多其他代码。

public class MapReduceIndexing extends Configured implements Tool{
static int number_of_documents;
static DynamoStorage ds = new DynamoStorage();  

public static class IndexMapper extends Mapper<AvroKey<DocumentSchema>, NullWritable, Text, IndexValue>{
    public void map(AvroKey<DocumentSchema> key, NullWritable value, Context context) throws IOException, InterruptedException {

        System.out.println("inside map start");

        //some mapper code e.g.
        for(String word : all_words.keySet()){
            context.write(new Text(word), iv);              
        }
        System.out.println("inside map end");
    }
}

public static class IndexReducer extends Reducer<Text, IndexValue, AvroKey<CharSequence>, AvroValue<Integer>> {

    @Override
    public void reduce(Text key, Iterable<IndexValue> iterable_values, Context context) throws IOException, InterruptedException {
        System.out.println("inside reduce start");
        //some reducer code
        System.out.println("inside reduce end");
    }
}

public int run(String[] args) throws Exception {        
    Configuration conf = new Configuration();       
    Job job = new Job(conf, "indexing");
    job.setJarByClass(MapReduceIndexing.class);
    job.setJobName("Making inverted index");

    FileInputFormat.setInputPaths(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setInputFormatClass(AvroKeyInputFormat.class);
    job.setMapperClass(IndexMapper.class);
    AvroJob.setInputKeySchema(job, DocumentSchema.getClassSchema());
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IndexValue.class);

    job.setOutputFormatClass(AvroKeyValueOutputFormat.class);
    job.setReducerClass(IndexReducer.class);
    AvroJob.setOutputKeySchema(job, Schema.create(Schema.Type.STRING));
    AvroJob.setOutputValueSchema(job, Schema.create(Schema.Type.INT));

    return (job.waitForCompletion(true) ? 0 : 1);
}

public static void main(String[] args) throws Exception {
    //setting input and output directories

    AWSCredentials credentials = new BasicAWSCredentials("access key", "secret key");
    AmazonS3 s3 = new AmazonS3Client(credentials);      
    ObjectListing object_listing = s3.listObjects(new ListObjectsRequest().withBucketName(args[2]));
    number_of_documents = object_listing.getObjectSummaries().size();

    int res = ToolRunner.run(new MapReduceIndexing(), args);
    System.exit(res);
}}
b1payxdu

b1payxdu1#

检查avro工具是否在编译类路径上。它包括一个 org.apache.hadoop.mapreduce.TaskAttemptContext 这可能与jar和/或集群中的版本冲突。如果您出于某种原因需要包含avro工具,那么您必须下载一个针对您的hadoop版本编译的版本(cloudera在其存储库中有这个版本,但我不确定从何处获得它以用于emr),或者自己编译avro工具。

相关问题