json处理

yeotifhr 于 2021-06-02 发布在 Hadoop

关注(0)|答案(3)|浏览(333)

我试图在json文件上运行map reduce。输入文件的格式如下。

{"Id":1, "title":"A list of SaaS management resources to help kickstart and augment your efforts","category":"business"}
{"Id":2, "title":"All Over the Board: 1Working on a  23 (Temp) Dream","category":"business"}
{"Id":3, "title":"Tulsa Web Design","category":"business"}

reduce函数的预期输出如下。

1 business A 1
1 business list 1
1 business of 1

下面是我用来读取json文件的代码，获取所需的值，然后将其转换为字符串。通过将字符串拆分为关键字值来对该字符串进行字数计算，关键字为id+category+标题中的每个单词，值为1。

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.json.*;

public class mr1 {

    public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable>{

        private final static IntWritable ONE = new IntWritable(1);
        private Text word = new Text();

        public void map(Object key, Text value, Context context) throws IOException, InterruptedException {

               String title;
                       String Id;
                       String category;
                       String line;
                       String valueLine;

            try {

                    line = value.toString();

                    JSONObject obj = new JSONObject(parser.parse(line));

                    title = (String) obj.get("title");
                    category = (String) obj.get("category");
                    Id = (String) obj.get("Id");

                    title = title.replaceAll("[!?,:()1-9]", "");
                    String[] strs = title.split("\\s+");

                    StringBuilder sb = new StringBuilder();

                            for(int i=0; i < strs.length; i++) {
                                sb.append(strs[i]+" ");
                            }

                    // valueLine  = 1 business Tulsa Web Design

                    valueLine = Id + " " + category + " " + sb.toString();

                    StringTokenizer itr = new StringTokenizer(valueLine);
                    String IndexAndCategory = "";

                    IndexAndCategory += itr.nextToken() + " ";
                    IndexAndCategory += itr.nextToken() + " ";

                    while(itr.hasMoreTokens()) {

                        word.set(IndexAndCategory + itr.nextToken());
                        context.write(word, ONE);
                    }

            } catch (JSONException e) {
                e.printStackTrace();
            }
        }

    }

    public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {

        private IntWritable result = new IntWritable();

        @Override
        public void reduce(Text key, Iterable<IntWritable> values, Context context)  
                throws IOException, InterruptedException {

                int sum = 0;
                for(IntWritable val : values) {     
                    sum += val.get();               
                }

                result.set(sum);
                context.write(key, result);
        }   
    }

    public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf, "mr1");
        job.setJarByClass(mr1.class);
        job.setMapperClass(TokenizerMapper.class);
        job.setCombinerClass(IntSumReducer.class);
                job.setReducerClass(IntSumReducer.class);
                job.setOutputKeyClass(Text.class);
                job.setOutputValueClass(IntWritable.class);
                job.setNumReduceTasks(1);
                FileInputFormat.addInputPath(job, new Path(args[0]));
                FileOutputFormat.setOutputPath(job, new Path(args[1]));
                System.exit(job.waitForCompletion(true) ? 0 : 1);

    }
}

我能够编译代码来创建jar文件，当我在hadoop上运行它时，我得到了以下错误。

Error: java.lang.ClassNotFoundException: org.json.JSONException
        at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
        at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
        at java.security.AccessController.doPrivileged(Native Method)
        at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
        at java.lang.Class.forName0(Native Method)
        at java.lang.Class.forName(Class.java:270)
        at org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:1986)
        at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:1951)
        at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2045)
        at org.apache.hadoop.mapreduce.task.JobContextImpl.getMapperClass(JobContextImpl.java:196)
        at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:742)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:168)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:415)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1614)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:163)

我用于此代码的jar文件如下所示；

1. hadoop-common-2.5.0.jar 
2. json-20160212.jar
3. hadoop-mapreduce-client-core-2.5.0.jar

我想问题是我将行转换成jsonobject的地方，但是我不确定这个问题。任何帮助解决这个问题都是非常感谢的。

Java hadoop JSON

来源：https://stackoverflow.com/questions/35475050/json-processing-on-hadoop

3条答案

按热度按时间

kdfy810k1#

你在用哪个ide？
1> 如果您使用的是netbeans，那么将所有库绑定到一个jar（胖jar）中。为此，只需在bulid.xml文件中包含一些xml代码

<target name="-post-jar">
    <property name="store.jar.name" value="NameOfYourJar"/>
    <property name="store.dir" value="dist"/>
    <property name="store.jar" value="${store.dir}/${store.jar.name}.jar"/>
    <echo message="Packaging ${application.title} into a single JAR at ${store.jar}"/>
    <jar destfile="${store.dir}/temp_final.jar" filesetmanifest="skip">
        <zipgroupfileset dir="dist" includes="*.jar"/>
        <zipgroupfileset dir="dist/lib" includes="*.jar"/>
        <manifest>
            <attribute name="Main-Class" value="${main.class}"/>
        </manifest>
    </jar>
    <zip destfile="${store.jar}">
        <zipfileset src="${store.dir}/temp_final.jar"
        excludes="META-INF/*.SF, META-INF/*.DSA, META-INF/*.RSA"/>
    </zip>
    <delete file="${store.dir}/temp_final.jar"/>
</target>

2> 如果您使用的是eclipse，那么就有点简单了

Right click the project 
Point to build path
and Export jar with all dependancy

然后用适当的参数将jar提交给hadoop，这可能会起作用，因为所有依赖项都绑定在一个jar中。你不必为类路径操心，你可以从任何hadoop安装的机器上提交jar(如果您使用的是maven build，则忽略）

赞(0）回复(0）举报 2021-06-02

u1ehiz5o2#

这是MapReduce作业中的Map器类(减速器等级同上）。

public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable>{

                private final static IntWritable ONE = new IntWritable(1);
                private Text word = new Text();

                public void map(Object key, Text value, Context context) throws IOException, InterruptedException, JsonParseException {

                           String title;
                           String id;
                           String cat;
                           String line;
                           String valueLine;
                           Details details = null;

                           line = value.toString();

                           // json object
                           Gson gsoObj = new Gson();
                           details = gsoObj.fromJson(line, Details.class);

                           // values
                           id = details.id;
                           cat = details.category;
                           title = details.title;

                           String final_ = id + " " +  cat + " " + title;

                           final_ = final_.replaceAll("[!?,:()1-9]", "");

                           // valueLine  = 1 business Tulsa Web Design

                           valueLine = final_;

                           StringTokenizer itr = new StringTokenizer(valueLine);
                           String IndexAndCategory = "";

                           IndexAndCategory += itr.nextToken() + " ";
                           IndexAndCategory += itr.nextToken() + " ";

                           while(itr.hasMoreTokens()) {

                            word.set(IndexAndCategory + itr.nextToken());
                            context.write(word, ONE);
                            }

                }

        private class Details {

                        protected String id = "";
                        protected String title = "";
                        protected String category = "";
                }

        }

我使用gson库gson-2.3.1.jar来处理json文件中的json数据。

赞(0）回复(0）举报 2021-06-02

lyr7nygr3#

请检查是否忘记在hadoop作业jar中添加json jar。这可能有助于您：http://tikalk.com/build-your-first-hadoop-project-maven

赞(0）回复(0）举报 2021-06-02