我尝试在hadoop多节点集群上实现eleuni\u max应用程序。但运行命令时出错:
$hadoop\u home/bin/hadoop jar units.jar hadoop.processunits input1\u dir output\u dir
…以下是源代码:
package hadoop;
import java.util.*;
import java.io.IOException;
import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.util.*;
public class ProcessUnits
{
//Mapper class
public static class E_EMapper extends MapReduceBase implements
Mapper<LongWritable ,/*Input key Type */
Text, /*Input value Type*/
Text, /*Output key Type*/
IntWritable> /*Output value Type*/
{
//Map function
public void map(LongWritable key, Text value,
OutputCollector<Text, IntWritable> output,
Reporter reporter) throws IOException
{
String line = value.toString();
String lasttoken = null;
StringTokenizer s = new StringTokenizer(line,"\t");
String year = s.nextToken();
while(s.hasMoreTokens())
{
lasttoken=s.nextToken();
}
int avgprice = Integer.parseInt(lasttoken);
output.collect(new Text(year), new IntWritable(avgprice));
}
}
//Reducer class
public static class E_EReduce extends MapReduceBase implements
Reducer< Text, IntWritable, Text, IntWritable >
{
//Reduce function
public void reduce( Text key, Iterator <IntWritable> values,
OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException
{
int maxavg=30;
int val=Integer.MIN_VALUE;
while (values.hasNext())
{
if((val=values.next().get())>maxavg)
{
output.collect(key, new IntWritable(val));
}
}
}
}
//Main function
public static void main(String args[])throws Exception
{
JobConf conf = new JobConf(ProcessUnits.class);
conf.setJobName("max_eletricityunits");
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(IntWritable.class);
conf.setMapperClass(E_EMapper.class);
conf.setCombinerClass(E_EReduce.class);
conf.setReducerClass(E_EReduce.class);
conf.setInputFormat(TextInputFormat.class);
conf.setOutputFormat(TextOutputFormat.class);
FileInputFormat.setInputPaths(conf, new Path(args[0]));
FileOutputFormat.setOutputPath(conf, new Path(args[1]));
JobClient.runJob(conf);
}
}
error :
hadoop@hadoop:~$ $HADOOP_HOME/bin/hadoop jar units.jar hadoop.ProcessUnits input1_dir output_dir
Warning: $HADOOP_HOME is deprecated.
Exception in thread "main" java.lang.ClassNotFoundException: hadoop.ProcessUnits
at java.net.URLClassLoader$1.run(URLClassLoader.java:359)
at java.net.URLClassLoader$1.run(URLClassLoader.java:348)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:347)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:278)
at org.apache.hadoop.util.RunJar.main(RunJar.java:153)
1条答案
按热度按时间gmxoilav1#
我复制了您的代码,并且能够开始处理,请检查您创建jar的方式。编译jar时可能没有包含主类。
你能试一下下面的东西吗,我可以用你的code:-
回音$javau回家。
export hadoop\u classpath=$java\u home/lib/tools.jar
创建包'hadoop'(mkdir hadoop)。(维护包名结构,以便hadoop理解)。
将“processunits.java”复制到hadoop文件夹中。
hadoop com.sun.tools.javac.main hadoop/processunits.java
jar-cf units.jar hadoop/processunits*.class
您的jar已创建。您可以运行以下命令:-hadoop jar units.jar hadoop/processunits inputdir outputdir